[help] Access UserName/Password restrict WebPages!

[wei liu]

hi everyone,

I am writing a crawler to download webpages but some pages are protected with username/password.

I am trying to use the authenticator to gain access to thoes pages provided with my username/password,but failed

here is my code below

   
package MiningExtract;

import java.net.Authenticator;
import java.net.InetAddress;
import java.net.PasswordAuthentication;

public class MyMinningAuthenticator extends Authenticator{
	public MyMinningAuthenticator(){
		System.out.println("MyMinningAuthenticator!!!");
	}
	protected  PasswordAuthentication getPasswordAuthentication() {
		System.out.println("getPasswordAuthentication!!!");
		//Get info about the request
		String promptString=getRequestingPrompt();
		String hostName=getRequestingHost();
		InetAddress ipaddr=getRequestingSite();
		int port = getRequestingPort();
		System.out.println("promptString is "+promptString);
		System.out.println("hostName is "+hostName);
		System.out.println("ipaddr is "+ipaddr);
		System.out.println("port is "+port);
		
		//get Username and password information
		String userName="debii";
		String password="debiidebii1";
		
		//Return the information
		
		return new PasswordAuthentication(userName,password.toCharArray());
		
	}
}

 

and part of code from my crawler

  
  public String saveToString(String destUrl){
			// Install the custom authenticator
			  
			  Authenticator.setDefault(new MyMinningAuthenticator());

		      String fileInString="";
			    HttpURLConnection httpUrl = null;
			    URL url = null;
			    byte[] buf = new byte[BUFFER_SIZE];
			    int size = 0;
			   
			    //setup connection
			    try {
				url = new URL(destUrl);
			    } catch (MalformedURLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			    }
			    try {
				httpUrl = (HttpURLConnection) url.openConnection();
			    } catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			    }
			    //System.out.println("HttpUrl OpenConnect OK!");
			    //connect
			    try {
				httpUrl.connect();
			    } catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			    }
			   
			   // System.out.println("HttpUrl Connect OK!");
			    //get input stream
			    
			    try {
				fileInString=this.inputStream2String(httpUrl.getInputStream());
			    } catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			    }
			    httpUrl.disconnect();
		      
		      
		      return fileInString;
		  }
 

I noticed by the result that only "MyMinningAuthenticator!!!" is printing out, but not the String "getPasswordAuthentication!!!"
is that means the protected method in MyMinningAuthenticatior is not running.


Thanks