tagList = tags.entrySet()
- .stream()
- .map(e -> Tag.builder().key(e.getKey()).value(e.getValue()).build())
- .collect(Collectors.toList());
-
- final PutObjectTaggingRequest request = PutObjectTaggingRequest.builder()
- .bucket(bucket)
- .key(objectName)
- .tagging(Tagging.builder().tagSet(tagList).build())
- .build();
- s3Client.putObjectTagging(request);
- } catch (final Exception e) {
- throw new StorageException("Failed to update tags for " + objectName, e);
- }
- }
-
- @Override
- public void ensureBucketExists() {
- try {
- final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
- s3Client.headBucket(request);
- } catch (final NoSuchBucketException e) {
- try {
- final CreateBucketRequest createRequest = CreateBucketRequest.builder().bucket(bucket).build();
- s3Client.createBucket(createRequest);
- logger.info("Created storage bucket: {}", bucket);
- } catch (final Exception e1) {
- logger.warn("Failed to create storage bucket: {}", bucket, e1);
- }
- } catch (final Exception e) {
- if (logger.isDebugEnabled()) {
- logger.debug("Failed to check bucket: {}", bucket, e);
- }
- }
- }
-
- @Override
- public boolean isAvailable() {
- try {
- final HeadBucketRequest request = HeadBucketRequest.builder().bucket(bucket).build();
- s3Client.headBucket(request);
- return true;
- } catch (final Exception e) {
- return false;
- }
- }
-
- @Override
- public void close() {
- if (s3Client != null) {
- s3Client.close();
- }
- }
-
- /**
- * Extracts the file/directory name from a full object path.
- *
- * @param objectName the full object path
- * @return the name portion of the path
- */
- private String getName(final String objectName) {
- if (StringUtil.isBlank(objectName)) {
- return StringUtil.EMPTY;
- }
- // Remove trailing slash if present
- String name = objectName;
- if (name.endsWith("/")) {
- name = name.substring(0, name.length() - 1);
- }
- final String[] values = name.split("/");
- if (values.length == 0) {
- return StringUtil.EMPTY;
- }
- return values[values.length - 1];
- }
-
- /**
- * Encodes an object name to a URL-safe base64 string.
- *
- * @param objectName the object name to encode
- * @return base64 encoded string
- */
- private String encodeId(final String objectName) {
- if (objectName == null) {
- return StringUtil.EMPTY;
- }
- return new String(Base64.getUrlEncoder().encode(objectName.getBytes(Constants.UTF_8_CHARSET)), Constants.UTF_8_CHARSET);
- }
-}
diff --git a/src/main/java/org/codelibs/fess/storage/StorageClientFactory.java b/src/main/java/org/codelibs/fess/storage/StorageClientFactory.java
index 14f371178..021e3c0f5 100644
--- a/src/main/java/org/codelibs/fess/storage/StorageClientFactory.java
+++ b/src/main/java/org/codelibs/fess/storage/StorageClientFactory.java
@@ -72,8 +72,10 @@ public static StorageType detectStorageType(final String endpoint) {
public static StorageClient createClient(final FessConfig fessConfig) {
final String componentName = componentName(fessConfig.getStorageType(), fessConfig.getStorageEndpoint());
if (!ComponentUtil.hasComponent(componentName)) {
- throw new StorageException("No storage client is registered as " + componentName + " for storage.type="
- + fessConfig.getStorageType() + ". Install the plugin that provides it, such as fess-storage-gcs for gcs.");
+ throw new StorageException(
+ "No storage client is registered as " + componentName + " for storage.type=" + fessConfig.getStorageType()
+ + ". Every backend ships as a fess-storage-* plugin: install fess-storage-s3 for s3 and s3_compat, "
+ + "or fess-storage-gcs for gcs.");
}
if (logger.isDebugEnabled()) {
logger.debug("Creating {} for endpoint: {}", componentName, fessConfig.getStorageEndpoint());
@@ -87,10 +89,10 @@ public static StorageClient createClient(final FessConfig fessConfig) {
* Returns the name of the DI component that serves a storage type.
*
* The mapping from a {@code storage.type} value to an implementation lives in the DI
- * definition rather than here, which is what lets the clients ship as plugins: core no longer
- * names GcsStorageClient or S3StorageClient, and a plugin registering
- * {@code StorageClient} is reachable by setting {@code storage.type=}. The
- * components are prototypes because every caller closes the client it was handed.
+ * definition rather than here, which is what lets the clients ship as plugins: core names no
+ * implementation at all, and a plugin registering {@code StorageClient} is reachable by
+ * setting {@code storage.type=}. The components are prototypes because every caller
+ * closes the client it was handed.
*
* @param typeStr the configured type, blank or {@code auto} to detect from the endpoint
* @param endpoint the storage endpoint, used only when detecting
diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties
index 930757415..066b66cda 100644
--- a/src/main/resources/fess_config.properties
+++ b/src/main/resources/fess_config.properties
@@ -383,7 +383,7 @@ crawler.crawling.data.encoding=UTF-8
# Supported web protocols for crawling.
crawler.web.protocols=http,https
# Supported file protocols for crawling.
-crawler.file.protocols=file,smb,smb1,ftp,s3
+crawler.file.protocols=file,smb,smb1,ftp
# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
diff --git a/src/main/resources/fess_storage.xml b/src/main/resources/fess_storage.xml
index 00846fbc8..faae617b6 100644
--- a/src/main/resources/fess_storage.xml
+++ b/src/main/resources/fess_storage.xml
@@ -2,21 +2,18 @@
-
-
-
-
-
-
diff --git a/src/test/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardActionTest.java b/src/test/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardActionTest.java
index 3c7568eb3..190ebf75c 100644
--- a/src/test/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardActionTest.java
+++ b/src/test/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardActionTest.java
@@ -42,7 +42,7 @@ public String getCrawlerWebProtocols() {
@Override
public String getCrawlerFileProtocols() {
- return "file,smb,smb1,ftp,s3";
+ return "file,smb,smb1,ftp";
}
});
final ProtocolHelper protocolHelper = new ProtocolHelper();
diff --git a/src/test/java/org/codelibs/fess/app/web/go/GoActionTest.java b/src/test/java/org/codelibs/fess/app/web/go/GoActionTest.java
index 97166ee3f..114a6205b 100644
--- a/src/test/java/org/codelibs/fess/app/web/go/GoActionTest.java
+++ b/src/test/java/org/codelibs/fess/app/web/go/GoActionTest.java
@@ -48,7 +48,7 @@ public String getCrawlerWebProtocols() {
@Override
public String getCrawlerFileProtocols() {
- return "file,smb,smb1,ftp,s3";
+ return "file,smb,smb1,ftp";
}
});
final ProtocolHelper protocolHelper = new ProtocolHelper();
diff --git a/src/test/java/org/codelibs/fess/crawler/transformer/FessFileTransformerTest.java b/src/test/java/org/codelibs/fess/crawler/transformer/FessFileTransformerTest.java
index ae37d5ab2..000eb934e 100644
--- a/src/test/java/org/codelibs/fess/crawler/transformer/FessFileTransformerTest.java
+++ b/src/test/java/org/codelibs/fess/crawler/transformer/FessFileTransformerTest.java
@@ -46,7 +46,7 @@ public String getCrawlerWebProtocols() {
@Override
public String getCrawlerFileProtocols() {
- return "file,smb,smb1,ftp,s3";
+ return "file,smb,smb1,ftp";
}
@Override
diff --git a/src/test/java/org/codelibs/fess/helper/ProtocolHelperTest.java b/src/test/java/org/codelibs/fess/helper/ProtocolHelperTest.java
index eb09cb026..08bf70936 100644
--- a/src/test/java/org/codelibs/fess/helper/ProtocolHelperTest.java
+++ b/src/test/java/org/codelibs/fess/helper/ProtocolHelperTest.java
@@ -738,24 +738,25 @@ public String getCrawlerWebProtocols() {
@Override
public String getCrawlerFileProtocols() {
- return "file,smb,smb1,ftp,s3";
+ return "file,smb,smb1,ftp";
}
});
final ProtocolHelper protocolHelper = new ProtocolHelper();
protocolHelper.init();
- assertEquals(5, protocolHelper.getFileProtocols().length);
+ assertEquals(4, protocolHelper.getFileProtocols().length);
// All file protocols should be valid
assertTrue(protocolHelper.isValidFileProtocol("file:///path/to/file"));
assertTrue(protocolHelper.isValidFileProtocol("smb://server/share"));
assertTrue(protocolHelper.isValidFileProtocol("smb1://server/share"));
assertTrue(protocolHelper.isValidFileProtocol("ftp://ftp.example.com/file"));
- assertTrue(protocolHelper.isValidFileProtocol("s3://bucket/key"));
- // gcs is not in the shipped default: fess-storage-gcs adds it with addFileProtocol when the
- // plugin is installed, which test_s3_gcs_protocols_add_dynamically covers.
+ // Neither object-storage protocol is in the shipped default: fess-storage-s3 and
+ // fess-storage-gcs add theirs with addFileProtocol when the plugin is installed, which
+ // test_s3_gcs_protocols_add_dynamically covers.
+ assertFalse(protocolHelper.isValidFileProtocol("s3://bucket/key"));
assertFalse(protocolHelper.isValidFileProtocol("gcs://bucket/object"));
// Web protocols should NOT be valid as file protocols
diff --git a/src/test/java/org/codelibs/fess/storage/StorageClientFactoryTest.java b/src/test/java/org/codelibs/fess/storage/StorageClientFactoryTest.java
index 92c7e6502..c998bac65 100644
--- a/src/test/java/org/codelibs/fess/storage/StorageClientFactoryTest.java
+++ b/src/test/java/org/codelibs/fess/storage/StorageClientFactoryTest.java
@@ -18,6 +18,12 @@
import org.codelibs.fess.unit.UnitFessTestCase;
import org.junit.jupiter.api.Test;
+/**
+ * What core still owns is the type-to-name mapping: {@link StorageClientFactory#componentName}
+ * and the endpoint detection behind it. The implementations moved to fess-storage-s3 and
+ * fess-storage-gcs together with their SDKs, so the component definitions and the prototype
+ * contract are verified in those plugins, not here.
+ */
public class StorageClientFactoryTest extends UnitFessTestCase {
@Test
@@ -49,42 +55,49 @@ public void test_componentName_detectsFromTheEndpoint() {
}
/**
- * Every name {@link #test_componentName_detectsFromTheEndpoint} and the storage.type values
- * the distribution serves can produce has to exist in fess_storage.xml, or the admin storage
- * screen fails at runtime for a configuration the UI offers.
+ * The distribution on its own serves no storage type: every backend ships as a
+ * fess-storage-* plugin that contributes its component through fess_storage++.xml, so
+ * fess_storage.xml declares nothing and storage.type is a configuration error until the
+ * matching plugin is installed. This asserts the absence, because registering any of these
+ * names in core again would pull an SDK back into the war - the AWS one for s3 and s3_compat,
+ * the Google Cloud Storage one for gcs.
+ *
+ * test_app.xml includes fess_storage.xml, so this runs against the file the distribution
+ * ships rather than against an empty container.
*/
@Test
- public void test_everyTypeTheDistributionServesHasAComponent() {
- for (final StorageType type : new StorageType[] { StorageType.S3, StorageType.S3_COMPAT }) {
+ public void test_noTypeIsServedByCore() {
+ for (final StorageType type : StorageType.values()) {
final String name = StorageClientFactory.componentName(type.name(), null);
- assertTrue(org.codelibs.fess.util.ComponentUtil.hasComponent(name), name + " is not registered in fess_storage.xml");
+ assertFalse(org.codelibs.fess.util.ComponentUtil.hasComponent(name),
+ name + " belongs to a fess-storage-* plugin and must not be declared in fess_storage.xml");
}
- assertTrue(org.codelibs.fess.util.ComponentUtil.hasComponent(StorageClientFactory.componentName("auto", null)));
+ // auto with no endpoint resolves to s3, which is a plugin as well.
+ assertFalse(org.codelibs.fess.util.ComponentUtil.hasComponent(StorageClientFactory.componentName("auto", null)));
}
/**
- * GCS is deliberately not in that list. Its client ships in fess-storage-gcs, together with the
- * Google Cloud Storage SDK the distribution no longer carries, so what core keeps is the name
- * the plugin registers under and the endpoint detection that produces it. Registering a
- * component here again would pull the SDK back into the war.
+ * With the clients gone, the mapping from an endpoint to a type name is what core contributes
+ * to reaching them, so it is fixed here as well as through {@link #componentName} above: a
+ * type that came out wrong would resolve a component name no installed plugin registers.
*/
@Test
- public void test_gcsIsServedByAPluginRatherThanCore() {
- assertEquals("gcsStorageClient", StorageClientFactory.componentName(StorageType.GCS.name(), null));
- assertFalse(org.codelibs.fess.util.ComponentUtil.hasComponent("gcsStorageClient"),
- "gcsStorageClient belongs to fess-storage-gcs, which contributes it through fess_storage++.xml");
- }
+ public void test_detectStorageType_mapsEndpointsToTypes() {
+ // No endpoint at all means AWS, which supplies its own.
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType(null));
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType(""));
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType(" "));
- /**
- * A prototype, not a singleton: every caller closes the client it was handed, so a shared
- * instance would be shut down for everybody by the first caller to finish.
- */
- @Test
- public void test_theComponentsArePrototypes() {
- final Object first = org.codelibs.fess.util.ComponentUtil.getComponent("s3StorageClient");
- final Object second = org.codelibs.fess.util.ComponentUtil.getComponent("s3StorageClient");
- assertNotNull(first);
- assertNotNull(second);
- assertFalse(first == second, "s3StorageClient must be instance=\"prototype\"");
+ assertEquals(StorageType.GCS, StorageClientFactory.detectStorageType("https://storage.googleapis.com"));
+ assertEquals(StorageType.GCS, StorageClientFactory.detectStorageType("https://STORAGE.GOOGLEAPIS.COM"));
+ assertEquals(StorageType.GCS, StorageClientFactory.detectStorageType("https://bucket.storage.cloud.google.com"));
+
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType("https://s3.us-east-1.amazonaws.com"));
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType("https://bucket.s3.amazonaws.com"));
+ assertEquals(StorageType.S3, StorageClientFactory.detectStorageType("https://s3-accelerate.amazonaws.com"));
+
+ // No vendor host: MinIO and the like, served by the same client as s3 under another name.
+ assertEquals(StorageType.S3_COMPAT, StorageClientFactory.detectStorageType("http://minio.internal:9000"));
+ assertEquals(StorageType.S3_COMPAT, StorageClientFactory.detectStorageType("https://objects.example.com"));
}
}
diff --git a/src/test/java/org/codelibs/fess/storage/StorageItemTest.java b/src/test/java/org/codelibs/fess/storage/StorageItemTest.java
index 7f6f0fcb5..4f3d6f90b 100644
--- a/src/test/java/org/codelibs/fess/storage/StorageItemTest.java
+++ b/src/test/java/org/codelibs/fess/storage/StorageItemTest.java
@@ -19,9 +19,11 @@
import java.time.ZonedDateTime;
import org.codelibs.fess.unit.UnitFessTestCase;
+import org.junit.jupiter.api.Test;
public class StorageItemTest extends UnitFessTestCase {
+ @Test
public void test_constructorAndGetters() {
final String name = "test-file.txt";
final String path = "/documents/";
@@ -40,6 +42,7 @@ public void test_constructorAndGetters() {
assertEquals(encodedId, item.getEncodedId());
}
+ @Test
public void test_directoryItem() {
final String name = "documents";
final String path = "/";
@@ -57,6 +60,7 @@ public void test_directoryItem() {
assertEquals(encodedId, item.getEncodedId());
}
+ @Test
public void test_fileItem() {
final ZonedDateTime now = ZonedDateTime.now();
final StorageItem item = new StorageItem("report.pdf", "/reports/2025/", false, 2048576L, now, "cmVwb3J0LnBkZg==");
@@ -68,6 +72,7 @@ public void test_fileItem() {
assertEquals(now, item.getLastModified());
}
+ @Test
public void test_nullValues() {
final StorageItem item = new StorageItem(null, null, false, 0L, null, null);
@@ -79,6 +84,7 @@ public void test_nullValues() {
assertNull(item.getEncodedId());
}
+ @Test
public void test_emptyName() {
final StorageItem item = new StorageItem("", "/path/", false, 100L, null, "");
@@ -87,6 +93,7 @@ public void test_emptyName() {
assertEquals("", item.getEncodedId());
}
+ @Test
public void test_largeFileSize() {
final long largeSize = 10737418240L; // 10 GB
final StorageItem item = new StorageItem("large-file.zip", "/backups/", false, largeSize, null, "bGFyZ2UtZmlsZS56aXA=");
@@ -94,6 +101,7 @@ public void test_largeFileSize() {
assertEquals(largeSize, item.getSize());
}
+ @Test
public void test_rootPath() {
final StorageItem item = new StorageItem("root-file.txt", "", false, 512L, null, "cm9vdC1maWxlLnR4dA==");
@@ -101,6 +109,7 @@ public void test_rootPath() {
assertEquals("", item.getPath());
}
+ @Test
public void test_deeplyNestedPath() {
final String deepPath = "/level1/level2/level3/level4/level5/";
final StorageItem item = new StorageItem("nested.txt", deepPath, false, 256L, null, "bmVzdGVkLnR4dA==");
@@ -108,6 +117,7 @@ public void test_deeplyNestedPath() {
assertEquals(deepPath, item.getPath());
}
+ @Test
public void test_specialCharactersInName() {
final String specialName = "file with spaces & symbols!@#.txt";
final StorageItem item = new StorageItem(specialName, "/special/", false, 100L, null, "c3BlY2lhbA==");
@@ -115,6 +125,7 @@ public void test_specialCharactersInName() {
assertEquals(specialName, item.getName());
}
+ @Test
public void test_unicodeInName() {
final String unicodeName = "ファイル名.txt";
final StorageItem item = new StorageItem(unicodeName, "/unicode/", false, 200L, null, "dW5pY29kZQ==");
diff --git a/src/test/java/org/codelibs/fess/storage/StorageTypeTest.java b/src/test/java/org/codelibs/fess/storage/StorageTypeTest.java
index 73901c7a0..917f6729b 100644
--- a/src/test/java/org/codelibs/fess/storage/StorageTypeTest.java
+++ b/src/test/java/org/codelibs/fess/storage/StorageTypeTest.java
@@ -16,39 +16,47 @@
package org.codelibs.fess.storage;
import org.codelibs.fess.unit.UnitFessTestCase;
+import org.junit.jupiter.api.Test;
public class StorageTypeTest extends UnitFessTestCase {
+ @Test
public void test_enumValues() {
final StorageType[] types = StorageType.values();
assertNotNull(types);
assertEquals(3, types.length);
}
+ @Test
public void test_s3Type() {
assertEquals(StorageType.S3, StorageType.valueOf("S3"));
}
+ @Test
public void test_gcsType() {
assertEquals(StorageType.GCS, StorageType.valueOf("GCS"));
}
+ @Test
public void test_s3CompatType() {
assertEquals(StorageType.S3_COMPAT, StorageType.valueOf("S3_COMPAT"));
}
+ @Test
public void test_valueOf() {
for (final StorageType type : StorageType.values()) {
assertEquals(type, StorageType.valueOf(type.name()));
}
}
+ @Test
public void test_ordinal() {
assertEquals(0, StorageType.S3.ordinal());
assertEquals(1, StorageType.GCS.ordinal());
assertEquals(2, StorageType.S3_COMPAT.ordinal());
}
+ @Test
public void test_name() {
assertEquals("S3", StorageType.S3.name());
assertEquals("GCS", StorageType.GCS.name());
diff --git a/src/test/resources/test_app.xml b/src/test/resources/test_app.xml
index ec499dfe6..a2171bded 100644
--- a/src/test/resources/test_app.xml
+++ b/src/test/resources/test_app.xml
@@ -5,11 +5,12 @@
-
+