feat(fs): let a file crawl leave symbolic links alone

fess-crawler/src/main/java/org/codelibs/fess/crawler/client/fs/FileSystemClient.java

@@ -76,6 +76,9 @@ public class FileSystemClient extends AbstractCrawlerClient {
     /** Key for file groups in metadata */
     public static final String FS_FILE_GROUPS = "fsFileGroups";
 
+    /** Init parameter deciding whether a symbolic link is followed. */
+    public static final String FOLLOW_SYMLINK_PROPERTY = "followSymlink";
+
     /** Character encoding for files */
     protected String charset = Constants.UTF_8;
 
@@ -86,6 +89,14 @@ public class FileSystemClient extends AbstractCrawlerClient {
     /** Flag to track initialization status */
     protected AtomicBoolean isInit = new AtomicBoolean(false);
 
+    /**
+     * Whether a symbolic link is followed. Following one indexes the file it points at a second
+     * time, under the configuration that reached the link rather than the one that covers the
+     * target, so the copy carries the roles of the link. Where a share is readable by more people
+     * than the tree it links into, that is a way around the roles on the target.
+     */
+    protected boolean followSymlink = true;
+
     /**
      * Constructs a new FileSystemClient.
      */
@@ -166,6 +177,12 @@ protected ResponseData getResponseData(final String uri, final boolean includeCo
                 responseData.setHttpStatusCode(Constants.NOT_FOUND_STATUS_CODE);
                 responseData.setCharSet(charset);
                 responseData.setContentLength(0);
+            } else if (isSkippedSymlink(file)) {
+                logger.info("Skipped a symbolic link: file={}", file.getAbsolutePath());
+                responseData.setHttpStatusCode(Constants.FORBIDDEN_STATUS_CODE);
+                responseData.setCharSet(charset);
+                responseData.setContentLength(0);
+                responseData.setMimeType(APPLICATION_OCTET_STREAM);
             } else if (file.isFile()) {
                 // check file size
                 responseData.setContentLength(file.length());
@@ -226,6 +243,10 @@ protected ResponseData getResponseData(final String uri, final boolean includeCo
                             logger.debug("Found {} child entries in directory: directory={}", files.length, file.getAbsolutePath());
                         }
                         for (final File f : files) {
+                            if (isSkippedSymlink(f)) {
+                                logger.info("Skipped a symbolic link: file={}", f.getAbsolutePath());
+                                continue;
+                            }
                             final String childUri = f.toURI().toASCIIString();
                             requestDataSet.add(RequestDataBuilder.newRequestData().get().url(childUri).build());
                         }
@@ -288,6 +309,44 @@ protected FileOwnerAttributeView parseFileOwnerAttribute(final ResponseData resp
         }
     }
 
+    @Override
+    public void init() {
+        super.init();
+        followSymlink = getInitParameter(FOLLOW_SYMLINK_PROPERTY, Boolean.valueOf(followSymlink), Boolean.class).booleanValue();
+        if (logger.isDebugEnabled()) {
+            logger.debug("followSymlink={}", followSymlink);
+        }
+    }
+
+    /**
+     * Determines whether the given file must be left alone because it is a symbolic link and
+     * links are not being followed.
+     *
+     * @param file the file to check
+     * @return true if the file is to be skipped
+     */
+    protected boolean isSkippedSymlink(final File file) {
+        return !followSymlink && Files.isSymbolicLink(file.toPath());
+    }
+
+    /**
+     * Returns whether a symbolic link is followed.
+     *
+     * @return true if symbolic links are followed
+     */
+    public boolean isFollowSymlink() {
+        return followSymlink;
+    }
+
+    /**
+     * Sets whether a symbolic link is followed.
+     *
+     * @param followSymlink true to follow symbolic links
+     */
+    public void setFollowSymlink(final boolean followSymlink) {
+        this.followSymlink = followSymlink;
+    }
+
     /**
      * Preprocesses a URI to ensure it's in the correct format for file system access.
      *

fess-crawler/src/test/java/org/codelibs/fess/crawler/client/fs/FileSystemClientTest.java

@@ -74,6 +74,70 @@ public void test_doGet_dir() {
 
     }
 
+    /**
+     * A symbolic link is followed by default, which is what java.io.File does and what every
+     * release so far has done.
+     */
+    @Test
+    public void test_followSymlink_isOnByDefault() {
+        assertTrue(fsClient.isFollowSymlink());
+    }
+
+    /**
+     * With links turned off, a link is not read. Following one indexes the file it points at a
+     * second time, under the configuration that reached the link rather than the one that covers
+     * the target, so the copy carries the roles of the link: on a share that more people can read
+     * than the tree it links into, that is a way around the roles on the target.
+     */
+    @Test
+    public void test_doGet_symlink_isRefusedWhenLinksAreNotFollowed() throws Exception {
+        final File target = ResourceUtil.getResourceAsFile("test/text1.txt");
+        final File link = new File(tempDir(), "link1.txt");
+        try {
+            Files.createSymbolicLink(link.toPath(), target.toPath());
+        } catch (final UnsupportedOperationException | java.io.IOException e) {
+            return; // the file system does not support symbolic links
+        }
+
+        fsClient.setFollowSymlink(false);
+        final ResponseData responseData = fsClient.doGet(link.toURI().toASCIIString());
+        assertEquals(Constants.FORBIDDEN_STATUS_CODE, responseData.getHttpStatusCode());
+        assertNull(responseData.getResponseBody());
+    }
+
+    /**
+     * A link inside a directory is left out of the child urls, so it is never queued either.
+     */
+    @Test
+    public void test_doGet_dir_leavesOutSymlinkChildrenWhenLinksAreNotFollowed() throws Exception {
+        final File target = ResourceUtil.getResourceAsFile("test/text1.txt");
+        final File dir = new File(tempDir(), "withlink");
+        assertTrue(dir.mkdirs() || dir.isDirectory());
+        final File link = new File(dir, "link2.txt");
+        try {
+            Files.createSymbolicLink(link.toPath(), target.toPath());
+        } catch (final UnsupportedOperationException | java.io.IOException e) {
+            return; // the file system does not support symbolic links
+        }
+
+        fsClient.setFollowSymlink(false);
+        try {
+            fsClient.doGet(dir.toURI().toASCIIString());
+            fail();
+        } catch (final ChildUrlsException e) {
+            for (final RequestData requestData : e.getChildUrlList()) {
+                assertFalse(requestData.getUrl().contains("link2.txt"));
+            }
+        }
+    }
+
+    private File tempDir() {
+        final File dir = new File(System.getProperty("java.io.tmpdir"), "fsClientSymlinkTest" + System.nanoTime());
+        assertTrue(dir.mkdirs());
+        dir.deleteOnExit();
+        return dir;
+    }
+
     @Test
     public void test_doGet_file() throws Exception {
         final File file = ResourceUtil.getResourceAsFile("test/text1.txt");