tag.
*
* @param anchor The anchor text to be analyzed.
*/
void analyzeAnchor(String anchor) {
String href = extract(anchor, "href");
// if (href == null) System.err.println("href null anchor" + anchor);
if (href == null) return;
try {
// href = fixUrl(href);
// System.err.println("in analyzeAnchore: adding " + base + "|" + href);
addURL(new URL(base, href));
} catch (MalformedURLException e) {
anchor = anchor.toLowerCase();
// java doesn't understand mailto and will throw an exception
// if (!href.startsWith("mailto:")) {
// e.printStackTrace();
// }
}
}
/**
* Analyzes the tag.
*
* @param frame The 'frame' tag being analyzed
*/
void analyzeFrame(String frame) {
String src = extract(frame, "src");
// if (src == null) System.err.println("src null");
if (src == null) return;
try {
// System.err.println("in analyzeFrame: adding " + base + "|" +
// src);
addURL(new URL(base, src));
} catch (MalformedURLException e) {
// e.printStackTrace();
}
}
/**
* Extracts the base URL from the tag.
*
* @param b Base tag, currently only works for 'href'
*/
void extractBase(String b) {
String b2 = extract(b, "href");
if (b2 != null) {
try {
base = new URL(base, b2);
} catch (MalformedURLException e) {
// e.printStackTrace();
}
}
}
/**
* Adds "url" to the list of URLs.
*
* @param url The url to be added
*/
public void addURL(URL url) {
urls.add(url);
url_count++;
}
/**
* Check to see if there is more elements.
*
* @return true if the list has more elements, false otherwise
*/
public boolean hasMoreElements() {
return url_count != next_url;
}
/**
* Return the next element in the list.
*
* @return The next element in the list
*/
public Object nextElement() {
Object ob = urls.get(next_url);
next_url++;
return ob;
}
/**
* Resets this enumeration.
*/
public void reset() {
next_url = 0;
}
/** Returns the value in "line" associated with "key", or null if "key"
* is not found. For instance, if line were "a href="blah blah blah"
* and "key" were "href" this method would return "blah blah blah".
*
* Keys are case insensitive.
*
* @param line The source data to work with
* @param key The pattern to look for
*/
String extract(String line, String key) {
// System.err.println("line in extract |" + line + "|");
line = line.replace('\'', '\"'); // some sites use ' instead of "
// System.err.println("line in extract after replace |" + line + "|");
try {
key = key.toLowerCase();
String lower_case = line.toLowerCase();
int i = lower_case.indexOf(key);
if (i < 0) return null;
i += key.length();
if (line.charAt(i) != '=') return null;
i++;
int i2;
if (line.charAt(i) == '"') {
i++;
i2 = line.indexOf('"', i);
if (i2 < 0) {
return line.substring(i);
} else {
return line.substring(i, i2);
}
} else {
int targ = line.length();
for (i2 = i; i < targ; i++) {
if (Character.isWhitespace(line.charAt(i))) break;
}
return line.substring(i, i2);
}
} catch (StringIndexOutOfBoundsException e) {}
return null;
}
/**
* Add '/' to a url whose ending is not a file.
* Then remove the extra '/'s.
* This is an add-hoc fix, only works for "href".
*
* @param inUrl The url to examine
* @return The fixed url
*/
static public String fixUrl(String inUrl)
{
int docLoc = inUrl.lastIndexOf('.');
int protLoc = inUrl.indexOf("://");
int slashLoc = -1;
if (protLoc > 0)
slashLoc = inUrl.indexOf('/', protLoc+3);
else
slashLoc = inUrl.indexOf('/');
if (slashLoc < 0 ||
(slashLoc > 0 && docLoc < slashLoc)) // none
inUrl = inUrl + '/';
inUrl = removeExtraSlash(inUrl);
return inUrl;
}
/**
* Remove extra '/'s in a url.
*
* @param inUrl The url to examine
* @return The fixed url
*/
static public String removeExtraSlash(String inStr)
{
int slashLoc = inStr.indexOf("://");
int l = inStr.length();
int hold = slashLoc + 3;
slashLoc = inStr.indexOf("//", hold);
while (slashLoc > 0 && slashLoc < l)
{
hold = slashLoc;
while (hold < l && inStr.charAt(hold) == '/')
hold ++;
if (hold >= l)
{
inStr = inStr.substring(0, slashLoc+1);
break;
}
else
inStr = inStr.substring(0, slashLoc+1)
+ inStr.substring(hold);
slashLoc = inStr.indexOf("//", hold);
l = inStr.length();
}
return inStr;
}
/**
* Print the string form of a url for a given url object.
*
* @param theUrl The url to be printed
*/
static public void printAUrl(URL theUrl)
{
if (theUrl.getPort() > 0)
System.out.println(theUrl.getProtocol() + "://" +
theUrl.getHost() + ":" +
theUrl.getPort() +
theUrl.getFile());
else if (theUrl.getProtocol().compareToIgnoreCase("mailto") == 0)
System.out.println(theUrl.getProtocol() + ":" +
theUrl.getFile());
else
System.out.println(theUrl.getProtocol() + "://" +
theUrl.getHost() +
theUrl.getFile());
}
/**
* Print all urls in the list.
*/
public void printAllUrls()
{
int count = urls.size();
for (int i = 0; i < count; i ++)
{
URL aUrl = (URL)urls.get(i);
printAUrl(aUrl);
}
}
/**
* Return all urls from the list.
*
* @return url list
*/
public ArrayList getAllUrls()
{
return urls;
}
/*
* Simple test drive.
*/
static public void main(String[] argv) throws Exception
{
if (argv.length != 2) {
System.err.println("usage: java HTMLLinkExtractor [url] [file-to-process]");
System.exit(1);
}
// argv[0] should be in the form of proto://host.domain/dir/file
URL aUrl = new URL(argv[0]);
File aFile = new File(argv[1]);
InputStream in = new FileInputStream(aFile);
String page = "";
int c = in.read();
while (c >= 0)
{
page += (char)c;
c = in.read();
}
HTMLLinkExtractor htmlExtractor =
new HTMLLinkExtractor(page, aUrl);
System.out.println("count of urls : " + htmlExtractor.urls.size());
htmlExtractor.printAllUrls();
}
}