NUTCH-3073 Address Java compiler warning

- replace (do not use) deprecated methods and fields
apache · Oct 4, 2024 · 1db4119 · 1db4119
1 parent 83405fb
commit 1db4119
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 9 deletions.
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -404,11 +404,6 @@ public void run() {
 
             switch (status.getCode()) {
 
-            case ProtocolStatus.WOULDBLOCK:
-              // retry ?
-              fetchQueues.addFetchItem(fit);
-              break;
-
             case ProtocolStatus.SUCCESS: // got a page
               pstatus = output(fit.url, fit.datum, content, status,
                   CrawlDatum.STATUS_FETCH_SUCCESS, fit.outlinkDepth);
@@ -457,8 +452,8 @@ public void run() {
                 context.getCounter("FetcherStatus",
                     "AboveExceptionThresholdInQueue").increment(killedURLs);
               /* FALLTHROUGH */
+
             case ProtocolStatus.RETRY: // retry
-            case ProtocolStatus.BLOCKED:
               output(fit.url, fit.datum, null, status,
                   CrawlDatum.STATUS_FETCH_RETRY);
               break;

diff --git a/...ocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthentication.java b/...ocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthentication.java
@@ -21,6 +21,7 @@
 import java.io.UnsupportedEncodingException;
 import java.net.CookieHandler;
 import java.net.CookieManager;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -132,7 +133,8 @@ public boolean getFollowRedirects() {
           LOG.debug("Response headers : " + header);
         }
       }
-      String rst = IOUtils.toString(post.getResponseBodyAsStream());
+      String rst = IOUtils.toString(post.getResponseBodyAsStream(),
+          StandardCharsets.UTF_8);
       LOG.debug("login post result: " + rst);
     } finally {
       if (post != null) {
@@ -194,7 +196,8 @@ private String httpGetPageContent(String url) throws IOException {
       if (cookieHeader != null) {
         setCookies(cookieHeader.getValue());
       }
-      String rst = IOUtils.toString(get.getResponseBodyAsStream());
+      String rst = IOUtils.toString(get.getResponseBodyAsStream(),
+          StandardCharsets.UTF_8);
       return rst;
     } finally {
       get.releaseConnection();

diff --git a/src/test/org/apache/nutch/fetcher/TestFetcher.java b/src/test/org/apache/nutch/fetcher/TestFetcher.java
@@ -100,7 +100,7 @@ public void testFetch() throws IOException, ClassNotFoundException, InterruptedE
     // generate
     Generator g = new Generator(conf);
     Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
-        Long.MAX_VALUE, Long.MAX_VALUE, false, false);
+        Long.MAX_VALUE, Long.MAX_VALUE, false, false, false, 1, null);
 
     long time = System.currentTimeMillis();
     // fetch