1 /***
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 package org.apache.nutch.parse.html;
19
20 import java.util.ArrayList;
21 import java.util.Map;
22 import java.net.URL;
23 import java.net.MalformedURLException;
24 import java.nio.charset.Charset;
25 import java.io.*;
26 import java.util.regex.*;
27
28 import org.cyberneko.html.parsers.*;
29 import org.xml.sax.InputSource;
30 import org.xml.sax.SAXException;
31 import org.w3c.dom.*;
32 import org.apache.html.dom.*;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36
37 import org.apache.nutch.metadata.Metadata;
38 import org.apache.nutch.metadata.Nutch;
39 import org.apache.nutch.protocol.Content;
40 import org.apache.hadoop.conf.*;
41 import org.apache.nutch.parse.*;
42 import org.apache.nutch.util.*;
43
44 public class HtmlParser implements Parser {
45 public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.html");
46
47 // I used 1000 bytes at first, but found that some documents have
48 // meta tag well past the first 1000 bytes.
49 // (e.g. http://cn.promo.yahoo.com/customcare/music.html)
50 private static final int CHUNK_SIZE = 2000;
51 private static Pattern metaPattern =
52 Pattern.compile("<meta\\s+([^>]*http-equiv=\"?content-type\"?[^>]*)>",
53 Pattern.CASE_INSENSITIVE);
54 private static Pattern charsetPattern =
55 Pattern.compile("charset=\\s*([a-z][_\\-0-9a-z]*)",
56 Pattern.CASE_INSENSITIVE);
57
58 private String parserImpl;
59
60 /***
61 * Given a <code>byte[]</code> representing an html file of an
62 * <em>unknown</em> encoding, read out 'charset' parameter in the meta tag
63 * from the first <code>CHUNK_SIZE</code> bytes.
64 * If there's no meta tag for Content-Type or no charset is specified,
65 * <code>null</code> is returned. <br />
66 * FIXME: non-byte oriented character encodings (UTF-16, UTF-32)
67 * can't be handled with this.
68 * We need to do something similar to what's done by mozilla
69 * (http://lxr.mozilla.org/seamonkey/source/parser/htmlparser/src/nsParser.cpp#1993).
70 * See also http://www.w3.org/TR/REC-xml/#sec-guessing
71 * <br />
72 *
73 * @param content <code>byte[]</code> representation of an html file
74 */
75
76 private static String sniffCharacterEncoding(byte[] content) {
77 int length = content.length < CHUNK_SIZE ?
78 content.length : CHUNK_SIZE;
79
80 // We don't care about non-ASCII parts so that it's sufficient
81 // to just inflate each byte to a 16-bit value by padding.
82 // For instance, the sequence {0x41, 0x82, 0xb7} will be turned into
83 // {U+0041, U+0082, U+00B7}.
84 String str = "";
85 try {
86 str = new String(content, 0, length,
87 Charset.forName("ASCII").toString());
88 } catch (UnsupportedEncodingException e) {
89 // code should never come here, but just in case...
90 return null;
91 }
92
93 Matcher metaMatcher = metaPattern.matcher(str);
94 String encoding = null;
95 if (metaMatcher.find()) {
96 Matcher charsetMatcher = charsetPattern.matcher(metaMatcher.group(1));
97 if (charsetMatcher.find())
98 encoding = new String(charsetMatcher.group(1));
99 }
100
101 return encoding;
102 }
103
104 private String defaultCharEncoding;
105
106 private Configuration conf;
107
108 private DOMContentUtils utils;
109
110 private HtmlParseFilters htmlParseFilters;
111
112 private String cachingPolicy;
113
114 public ParseResult getParse(Content content) {
115 HTMLMetaTags metaTags = new HTMLMetaTags();
116
117 URL base;
118 try {
119 base = new URL(content.getBaseUrl());
120 } catch (MalformedURLException e) {
121 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
122 }
123
124 String text = "";
125 String title = "";
126 Outlink[] outlinks = new Outlink[0];
127 Metadata metadata = new Metadata();
128
129 // parse the content
130 DocumentFragment root;
131 try {
132 byte[] contentInOctets = content.getContent();
133 InputSource input = new InputSource(new ByteArrayInputStream(contentInOctets));
134
135 EncodingDetector detector = new EncodingDetector(conf);
136 detector.autoDetectClues(content, true);
137 detector.addClue(sniffCharacterEncoding(contentInOctets), "sniffed");
138 String encoding = detector.guessEncoding(content, defaultCharEncoding);
139
140 metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding);
141 metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding);
142
143 input.setEncoding(encoding);
144 if (LOG.isTraceEnabled()) { LOG.trace("Parsing..."); }
145 root = parse(input);
146 } catch (IOException e) {
147 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
148 } catch (DOMException e) {
149 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
150 } catch (SAXException e) {
151 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
152 } catch (Exception e) {
153 e.printStackTrace(LogUtil.getWarnStream(LOG));
154 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
155 }
156
157 // get meta directives
158 HTMLMetaProcessor.getMetaTags(metaTags, root, base);
159 if (LOG.isTraceEnabled()) {
160 LOG.trace("Meta tags for " + base + ": " + metaTags.toString());
161 }
162 // check meta directives
163 if (!metaTags.getNoIndex()) { // okay to index
164 StringBuffer sb = new StringBuffer();
165 if (LOG.isTraceEnabled()) { LOG.trace("Getting text..."); }
166 utils.getText(sb, root); // extract text
167 text = sb.toString();
168 sb.setLength(0);
169 if (LOG.isTraceEnabled()) { LOG.trace("Getting title..."); }
170 utils.getTitle(sb, root); // extract title
171 title = sb.toString().trim();
172 }
173
174 if (!metaTags.getNoFollow()) { // okay to follow links
175 ArrayList<Outlink> l = new ArrayList<Outlink>(); // extract outlinks
176 URL baseTag = utils.getBase(root);
177 if (LOG.isTraceEnabled()) { LOG.trace("Getting links..."); }
178 utils.getOutlinks(baseTag!=null?baseTag:base, l, root);
179 outlinks = l.toArray(new Outlink[l.size()]);
180 if (LOG.isTraceEnabled()) {
181 LOG.trace("found "+outlinks.length+" outlinks in "+content.getUrl());
182 }
183 }
184
185 ParseStatus status = new ParseStatus(ParseStatus.SUCCESS);
186 if (metaTags.getRefresh()) {
187 status.setMinorCode(ParseStatus.SUCCESS_REDIRECT);
188 status.setArgs(new String[] {metaTags.getRefreshHref().toString(),
189 Integer.toString(metaTags.getRefreshTime())});
190 }
191 ParseData parseData = new ParseData(status, title, outlinks,
192 content.getMetadata(), metadata);
193 ParseResult parseResult = ParseResult.createParseResult(content.getUrl(),
194 new ParseImpl(text, parseData));
195
196 // run filters on parse
197 ParseResult filteredParse = this.htmlParseFilters.filter(content, parseResult,
198 metaTags, root);
199 if (metaTags.getNoCache()) { // not okay to cache
200 for (Map.Entry<org.apache.hadoop.io.Text, Parse> entry : filteredParse)
201 entry.getValue().getData().getParseMeta().set(Nutch.CACHING_FORBIDDEN_KEY,
202 cachingPolicy);
203 }
204 return filteredParse;
205 }
206
207 private DocumentFragment parse(InputSource input) throws Exception {
208 if (parserImpl.equalsIgnoreCase("tagsoup"))
209 return parseTagSoup(input);
210 else return parseNeko(input);
211 }
212
213 private DocumentFragment parseTagSoup(InputSource input) throws Exception {
214 HTMLDocumentImpl doc = new HTMLDocumentImpl();
215 DocumentFragment frag = doc.createDocumentFragment();
216 DOMBuilder builder = new DOMBuilder(doc, frag);
217 org.ccil.cowan.tagsoup.Parser reader = new org.ccil.cowan.tagsoup.Parser();
218 reader.setContentHandler(builder);
219 reader.setFeature(org.ccil.cowan.tagsoup.Parser.ignoreBogonsFeature, true);
220 reader.setFeature(org.ccil.cowan.tagsoup.Parser.bogonsEmptyFeature, false);
221 reader.setProperty("http://xml.org/sax/properties/lexical-handler", builder);
222 reader.parse(input);
223 return frag;
224 }
225
226 private DocumentFragment parseNeko(InputSource input) throws Exception {
227 DOMFragmentParser parser = new DOMFragmentParser();
228 try {
229 parser.setFeature("http://cyberneko.org/html/features/augmentations",
230 true);
231 parser.setProperty("http://cyberneko.org/html/properties/default-encoding",
232 defaultCharEncoding);
233 parser.setFeature("http://cyberneko.org/html/features/scanner/ignore-specified-charset",
234 true);
235 parser.setFeature("http://cyberneko.org/html/features/balance-tags/ignore-outside-content",
236 false);
237 parser.setFeature("http://cyberneko.org/html/features/balance-tags/document-fragment",
238 true);
239 parser.setFeature("http://cyberneko.org/html/features/report-errors",
240 LOG.isTraceEnabled());
241 } catch (SAXException e) {}
242 // convert Document to DocumentFragment
243 HTMLDocumentImpl doc = new HTMLDocumentImpl();
244 doc.setErrorChecking(false);
245 DocumentFragment res = doc.createDocumentFragment();
246 DocumentFragment frag = doc.createDocumentFragment();
247 parser.parse(input, frag);
248 res.appendChild(frag);
249
250 try {
251 while(true) {
252 frag = doc.createDocumentFragment();
253 parser.parse(input, frag);
254 if (!frag.hasChildNodes()) break;
255 if (LOG.isInfoEnabled()) {
256 LOG.info(" - new frag, " + frag.getChildNodes().getLength() + " nodes.");
257 }
258 res.appendChild(frag);
259 }
260 } catch (Exception x) { x.printStackTrace(LogUtil.getWarnStream(LOG));};
261 return res;
262 }
263
264 public static void main(String[] args) throws Exception {
265 //LOG.setLevel(Level.FINE);
266 String name = args[0];
267 String url = "file:"+name;
268 File file = new File(name);
269 byte[] bytes = new byte[(int)file.length()];
270 DataInputStream in = new DataInputStream(new FileInputStream(file));
271 in.readFully(bytes);
272 Configuration conf = NutchConfiguration.create();
273 HtmlParser parser = new HtmlParser();
274 parser.setConf(conf);
275 Parse parse = parser.getParse(
276 new Content(url, url, bytes, "text/html", new Metadata(), conf)).get(url);
277 System.out.println("data: "+parse.getData());
278
279 System.out.println("text: "+parse.getText());
280
281 }
282
283 public void setConf(Configuration conf) {
284 this.conf = conf;
285 this.htmlParseFilters = new HtmlParseFilters(getConf());
286 this.parserImpl = getConf().get("parser.html.impl", "neko");
287 this.defaultCharEncoding = getConf().get(
288 "parser.character.encoding.default", "windows-1252");
289 this.utils = new DOMContentUtils(conf);
290 this.cachingPolicy = getConf().get("parser.caching.forbidden.policy",
291 Nutch.CACHING_FORBIDDEN_CONTENT);
292 }
293
294 public Configuration getConf() {
295 return this.conf;
296 }
297 }
浙公网安备 33010602011771号