feat: support tencent cos

plugins/s3/build.gradle.kts

@@ -3,7 +3,7 @@ plugins {
 }
 
 
-project.version = "0.0.4"
+project.version = "0.0.5"
 
 
 dependencies {

plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystem.kt

@@ -0,0 +1,13 @@
+package app.termora.plugins.s3
+
+import app.termora.transfer.s3.S3FileSystem
+import io.minio.MinioClient
+
+class MyS3FileSystem(private val minioClient: MinioClient) :
+    S3FileSystem(MyS3FileSystemProvider(minioClient)) {
+
+    override fun close() {
+        minioClient.close()
+        super.close()
+    }
+}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystemProvider.kt

@@ -0,0 +1,157 @@
+package app.termora.plugins.s3
+
+import app.termora.transfer.s3.S3FileAttributes
+import app.termora.transfer.s3.S3FileSystemProvider
+import app.termora.transfer.s3.S3Path
+import io.minio.*
+import io.minio.errors.ErrorResponseException
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang3.StringUtils
+import java.io.InputStream
+import java.io.OutputStream
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.nio.file.AccessMode
+import java.nio.file.NoSuchFileException
+import java.util.concurrent.atomic.AtomicReference
+import kotlin.io.path.absolutePathString
+
+class MyS3FileSystemProvider(private val minioClient: MinioClient) : S3FileSystemProvider() {
+
+    override fun getScheme(): String? {
+        return "s3"
+    }
+
+    override fun getOutputStream(path: S3Path): OutputStream {
+        return createStreamer(path)
+    }
+
+    override fun getInputStream(path: S3Path): InputStream {
+        return minioClient.getObject(
+            GetObjectArgs.builder().bucket(path.bucketName)
+                .`object`(path.objectName).build()
+        )
+    }
+
+
+    private fun createStreamer(path: S3Path): OutputStream {
+        val pis = PipedInputStream()
+        val pos = PipedOutputStream(pis)
+        val exception = AtomicReference<Throwable>()
+
+        val thread = Thread.ofVirtual().start {
+            try {
+                minioClient.putObject(
+                    PutObjectArgs.builder()
+                        .bucket(path.bucketName)
+                        .stream(pis, -1, 32 * 1024 * 1024)
+                        .`object`(path.objectName).build()
+                )
+            } catch (e: Exception) {
+                exception.set(e)
+            } finally {
+                IOUtils.closeQuietly(pis)
+            }
+        }
+
+        return object : OutputStream() {
+            override fun write(b: Int) {
+                val exception = exception.get()
+                if (exception != null) throw exception
+                pos.write(b)
+            }
+
+            override fun close() {
+                pos.close()
+                if (thread.isAlive) thread.join()
+            }
+        }
+    }
+
+    override fun fetchChildren(path: S3Path): MutableList<S3Path> {
+        val paths = mutableListOf<S3Path>()
+
+        // root
+        if (path.isRoot) {
+            for (bucket in minioClient.listBuckets()) {
+                val p = path.resolve(bucket.name())
+                p.attributes = S3FileAttributes(
+                    directory = true,
+                    lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
+                )
+                paths.add(p)
+            }
+            return paths
+        }
+
+        var startAfter = StringUtils.EMPTY
+        val maxKeys = 100
+
+        while (true) {
+            val builder = ListObjectsArgs.builder()
+                .bucket(path.bucketName)
+                .maxKeys(maxKeys)
+                .delimiter(path.fileSystem.separator)
+
+            if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
+            if (startAfter.isNotBlank()) builder.startAfter(startAfter)
+
+
+            val subPaths = mutableListOf<S3Path>()
+            for (e in minioClient.listObjects(builder.build())) {
+                val item = e.get()
+                val p = path.bucket.resolve(item.objectName())
+                var attributes = p.attributes.copy(
+                    directory = item.isDir,
+                    regularFile = item.isDir.not(),
+                    size = item.size()
+                )
+                if (item.lastModified() != null) {
+                    attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
+                }
+                p.attributes = attributes
+
+                // 如果是文件夹，那么就要删除内存中的
+                if (attributes.isDirectory) {
+                    delete(p)
+                }
+
+                subPaths.add(p)
+                startAfter = item.objectName()
+            }
+
+            paths.addAll(subPaths)
+
+            if (subPaths.size < maxKeys)
+                break
+
+
+        }
+
+        paths.addAll(directories[path.absolutePathString()] ?: emptyList())
+
+        return paths
+    }
+
+
+    override fun delete(path: S3Path, isDirectory: Boolean) {
+        if (isDirectory.not())
+            minioClient.removeObject(
+                RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
+            )
+    }
+
+
+    override fun checkAccess(path: S3Path, vararg modes: AccessMode) {
+        try {
+            minioClient.statObject(
+                StatObjectArgs.builder()
+                    .`object`(path.objectName)
+                    .bucket(path.bucketName).build()
+            )
+        } catch (e: ErrorResponseException) {
+            throw NoSuchFileException(e.errorResponse().message())
+        }
+    }
+
+}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileAttributes.kt

@@ -1,52 +0,0 @@
-package app.termora.plugins.s3
-
-import java.nio.file.attribute.BasicFileAttributes
-import java.nio.file.attribute.FileTime
-
-data class S3FileAttributes(
-    private val lastModifiedTime: Long = 0,
-    private val lastAccessTime: Long = 0,
-    private val creationTime: Long = 0,
-
-    private val regularFile: Boolean = false,
-    private val directory: Boolean = false,
-    private val symbolicLink: Boolean = false,
-    private val other: Boolean = false,
-    private val size: Long = 0,
-) : BasicFileAttributes {
-    override fun lastModifiedTime(): FileTime {
-        return FileTime.fromMillis(lastModifiedTime)
-    }
-
-    override fun lastAccessTime(): FileTime {
-        return FileTime.fromMillis(lastAccessTime)
-    }
-
-    override fun creationTime(): FileTime {
-        return FileTime.fromMillis(creationTime)
-    }
-
-    override fun isRegularFile(): Boolean {
-        return regularFile
-    }
-
-    override fun isDirectory(): Boolean {
-        return directory
-    }
-
-    override fun isSymbolicLink(): Boolean {
-        return symbolicLink
-    }
-
-    override fun isOther(): Boolean {
-        return other
-    }
-
-    override fun size(): Long {
-        return size
-    }
-
-    override fun fileKey(): Any? {
-        return null
-    }
-}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystem.kt

@@ -1,44 +0,0 @@
-package app.termora.plugins.s3
-
-import io.minio.MinioClient
-import org.apache.sshd.common.file.util.BaseFileSystem
-import java.nio.file.Path
-import java.nio.file.attribute.UserPrincipalLookupService
-import java.util.concurrent.atomic.AtomicBoolean
-
-class S3FileSystem(
-    private val minioClient: MinioClient,
-) : BaseFileSystem<S3Path>(S3FileSystemProvider(minioClient)) {
-
-    private val isOpen = AtomicBoolean(true)
-
-    override fun create(root: String?, names: List<String>): S3Path {
-        val path = S3Path(this, root, names)
-        if (names.isEmpty()) {
-            path.attributes = path.attributes.copy(directory = true)
-        }
-        return path
-    }
-
-    override fun close() {
-        if (isOpen.compareAndSet(false, true)) {
-            minioClient.close()
-        }
-    }
-
-    override fun isOpen(): Boolean {
-        return isOpen.get()
-    }
-
-    override fun getRootDirectories(): Iterable<Path> {
-        return mutableSetOf<Path>(create(separator))
-    }
-
-    override fun supportedFileAttributeViews(): Set<String> {
-        TODO("Not yet implemented")
-    }
-
-    override fun getUserPrincipalLookupService(): UserPrincipalLookupService {
-        throw UnsupportedOperationException()
-    }
-}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystemProvider.kt

@@ -1,308 +0,0 @@
-package app.termora.plugins.s3
-
-import io.minio.*
-import io.minio.errors.ErrorResponseException
-import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.StringUtils
-import java.io.OutputStream
-import java.io.PipedInputStream
-import java.io.PipedOutputStream
-import java.net.URI
-import java.nio.channels.Channels
-import java.nio.channels.SeekableByteChannel
-import java.nio.file.*
-import java.nio.file.attribute.*
-import java.nio.file.spi.FileSystemProvider
-import java.util.concurrent.atomic.AtomicReference
-import kotlin.io.path.absolutePathString
-import kotlin.io.path.name
-
-class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemProvider() {
-
-    /**
-     * 因为 S3 协议不存在文件夹，所以用户新建的文件夹先保存到内存中
-     */
-    private val directories = mutableMapOf<String, MutableList<S3Path>>()
-
-    override fun getScheme(): String? {
-        return "s3"
-    }
-
-    override fun newFileSystem(
-        uri: URI,
-        env: Map<String, *>
-    ): FileSystem {
-        TODO("Not yet implemented")
-    }
-
-    override fun getFileSystem(uri: URI): FileSystem {
-        TODO("Not yet implemented")
-    }
-
-    override fun getPath(uri: URI): Path {
-        TODO("Not yet implemented")
-    }
-
-    override fun newByteChannel(
-        path: Path,
-        options: Set<OpenOption>,
-        vararg attrs: FileAttribute<*>
-    ): SeekableByteChannel {
-        if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
-        if (options.contains(StandardOpenOption.WRITE)) {
-            return S3WriteSeekableByteChannel(Channels.newChannel(createStreamer(path)))
-        } else {
-            val response = minioClient.getObject(
-                GetObjectArgs.builder().bucket(path.bucketName)
-                    .`object`(path.objectName).build()
-            )
-            return S3ReadSeekableByteChannel(Channels.newChannel(response), stat(path))
-        }
-    }
-
-
-    private fun createStreamer(path: S3Path): OutputStream {
-        val pis = PipedInputStream()
-        val pos = PipedOutputStream(pis)
-        val exception = AtomicReference<Throwable>()
-
-        val thread = Thread.ofVirtual().start {
-            try {
-                minioClient.putObject(
-                    PutObjectArgs.builder()
-                        .bucket(path.bucketName)
-                        .stream(pis, -1, 32 * 1024 * 1024)
-                        .`object`(path.objectName).build()
-                )
-            } catch (e: Exception) {
-                exception.set(e)
-            } finally {
-                IOUtils.closeQuietly(pis)
-            }
-        }
-
-        return object : OutputStream() {
-            override fun write(b: Int) {
-                val exception = exception.get()
-                if (exception != null) throw exception
-                pos.write(b)
-            }
-
-            override fun close() {
-                pos.close()
-                if (thread.isAlive) thread.join()
-            }
-        }
-    }
-
-    override fun newDirectoryStream(
-        dir: Path,
-        filter: DirectoryStream.Filter<in Path>
-    ): DirectoryStream<Path> {
-        return object : DirectoryStream<Path> {
-            override fun iterator(): MutableIterator<Path> {
-                return files(dir as S3Path).iterator()
-            }
-
-            override fun close() {
-            }
-
-        }
-    }
-
-    private fun files(path: S3Path): MutableList<S3Path> {
-        val paths = mutableListOf<S3Path>()
-
-        // root
-        if (path.isRoot) {
-            for (bucket in minioClient.listBuckets()) {
-                val p = path.resolve(bucket.name())
-                p.attributes = S3FileAttributes(
-                    directory = true,
-                    lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
-                )
-                paths.add(p)
-            }
-            return paths
-        }
-
-        var startAfter = StringUtils.EMPTY
-        val maxKeys = 100
-
-        while (true) {
-            val builder = ListObjectsArgs.builder()
-                .bucket(path.bucketName)
-                .maxKeys(maxKeys)
-                .delimiter(path.fileSystem.separator)
-
-            if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
-            if (startAfter.isNotBlank()) builder.startAfter(startAfter)
-
-
-            val subPaths = mutableListOf<S3Path>()
-            for (e in minioClient.listObjects(builder.build())) {
-                val item = e.get()
-                val p = path.bucket.resolve(item.objectName())
-                var attributes = p.attributes.copy(
-                    directory = item.isDir,
-                    regularFile = item.isDir.not(),
-                    size = item.size()
-                )
-                if (item.lastModified() != null) {
-                    attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
-                }
-                p.attributes = attributes
-
-                // 如果是文件夹，那么就要删除内存中的
-                if (attributes.isDirectory) {
-                    delete(p)
-                }
-
-                subPaths.add(p)
-                startAfter = item.objectName()
-            }
-
-            paths.addAll(subPaths)
-
-            if (subPaths.size < maxKeys)
-                break
-
-
-        }
-
-        paths.addAll(directories[path.absolutePathString()] ?: emptyList())
-
-        return paths
-    }
-
-    override fun createDirectory(dir: Path, vararg attrs: FileAttribute<*>) {
-        synchronized(this) {
-            if (dir !is S3Path) throw UnsupportedOperationException("dir must be a S3Path")
-            if (dir.isRoot || dir.isBucket) throw UnsupportedOperationException("No operation permission")
-            val parent = dir.parent ?: throw UnsupportedOperationException("No operation permission")
-            directories.computeIfAbsent(parent.absolutePathString()) { mutableListOf() }
-                .add(dir.apply {
-                    attributes = attributes.copy(directory = true, lastModifiedTime = System.currentTimeMillis())
-                })
-        }
-    }
-
-    override fun delete(path: Path) {
-        if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
-        if (path.attributes.isDirectory) {
-            val parent = path.parent
-            if (parent != null) {
-                synchronized(this) {
-                    directories[parent.absolutePathString()]?.removeIf { it.name == path.name }
-                }
-            }
-            return
-        }
-        minioClient.removeObject(
-            RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
-        )
-    }
-
-    override fun copy(
-        source: Path?,
-        target: Path?,
-        vararg options: CopyOption?
-    ) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun move(
-        source: Path?,
-        target: Path?,
-        vararg options: CopyOption?
-    ) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun isSameFile(path: Path?, path2: Path?): Boolean {
-        throw UnsupportedOperationException()
-    }
-
-    override fun isHidden(path: Path?): Boolean {
-        throw UnsupportedOperationException()
-    }
-
-    override fun getFileStore(path: Path?): FileStore? {
-        throw UnsupportedOperationException()
-    }
-
-    override fun checkAccess(path: Path, vararg modes: AccessMode) {
-        if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
-
-        try {
-            stat(path)
-        } catch (e: ErrorResponseException) {
-            throw NoSuchFileException(e.errorResponse().message())
-        }
-    }
-
-
-    private fun stat(path: S3Path): StatObjectResponse {
-        return minioClient.statObject(
-            StatObjectArgs.builder()
-                .`object`(path.objectName)
-                .bucket(path.bucketName).build()
-        )
-    }
-
-    override fun <V : FileAttributeView> getFileAttributeView(
-        path: Path,
-        type: Class<V>,
-        vararg options: LinkOption?
-    ): V {
-        if (path is S3Path) {
-            return type.cast(object : BasicFileAttributeView {
-                override fun name(): String {
-                    return "basic"
-                }
-
-                override fun readAttributes(): BasicFileAttributes {
-                    return path.attributes
-                }
-
-                override fun setTimes(
-                    lastModifiedTime: FileTime?,
-                    lastAccessTime: FileTime?,
-                    createTime: FileTime?
-                ) {
-                    throw UnsupportedOperationException()
-                }
-
-            })
-        }
-        throw UnsupportedOperationException()
-    }
-
-    override fun <A : BasicFileAttributes> readAttributes(
-        path: Path,
-        type: Class<A>,
-        vararg options: LinkOption
-    ): A {
-        if (path is S3Path) {
-            return type.cast(getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes())
-        }
-        throw UnsupportedOperationException()
-    }
-
-    override fun readAttributes(
-        path: Path?,
-        attributes: String?,
-        vararg options: LinkOption?
-    ): Map<String?, Any?>? {
-        throw UnsupportedOperationException()
-    }
-
-    override fun setAttribute(
-        path: Path?,
-        attribute: String?,
-        value: Any?,
-        vararg options: LinkOption?
-    ) {
-        throw UnsupportedOperationException()
-    }
-}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3Path.kt

@@ -1,60 +0,0 @@
-package app.termora.plugins.s3
-
-import app.termora.transfer.WithPathAttributes
-import org.apache.sshd.common.file.util.BasePath
-import java.nio.file.LinkOption
-import java.nio.file.Path
-import kotlin.io.path.absolutePathString
-
-class S3Path(
-    fileSystem: S3FileSystem,
-    root: String?,
-    names: List<String>,
-) : BasePath<S3Path, S3FileSystem>(fileSystem, root, names), WithPathAttributes {
-
-
-    private val separator get() = fileSystem.separator
-
-    var attributes = S3FileAttributes()
-
-    /**
-     * 是否是 Bucket
-     */
-    val isBucket get() = parent != null && parent?.parent == null
-
-    /**
-     * 是否是根
-     */
-    val isRoot get() = absolutePathString() == separator
-
-    /**
-     * Bucket Name
-     */
-    val bucketName: String get() = names.first()
-
-    /**
-     * 获取 Bucket
-     */
-    val bucket: S3Path get() = fileSystem.getPath(root, bucketName)
-
-    /**
-     * 获取所在 Bucket 的路径
-     */
-    val objectName: String get() = names.subList(1, names.size).joinToString(separator)
-
-    override fun getCustomType(): String? {
-        if (isBucket) return "Bucket"
-        return null
-    }
-
-    override fun toRealPath(vararg options: LinkOption): Path {
-        return toAbsolutePath()
-    }
-
-    override fun getParent(): S3Path? {
-        val path = super.getParent() ?: return null
-        path.attributes = path.attributes.copy(directory = true)
-        return path
-    }
-
-}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ProtocolProvider.kt

@@ -70,7 +70,7 @@ class S3ProtocolProvider private constructor() : TransferProtocolProvider {
 //        val delimiter = host.options.extras["s3.delimiter"] ?: "/"
         val defaultPath = host.options.sftpDefaultDirectory
         val minioClient = builder.build()
-        val fs = S3FileSystem(minioClient)
+        val fs = MyS3FileSystem(minioClient)
         return PathHandler(fs, fs.getPath(defaultPath))
     }
 

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ReadSeekableByteChannel.kt

@@ -1,51 +0,0 @@
-package app.termora.plugins.s3
-
-import io.minio.StatObjectResponse
-import org.apache.commons.io.IOUtils
-import java.nio.ByteBuffer
-import java.nio.channels.ReadableByteChannel
-import java.nio.channels.SeekableByteChannel
-
-class S3ReadSeekableByteChannel(
-    private val channel: ReadableByteChannel,
-    private val stat: StatObjectResponse
-) : SeekableByteChannel {
-
-    private var position: Long = 0
-
-    override fun read(dst: ByteBuffer): Int {
-        val bytesRead = channel.read(dst)
-        if (bytesRead > 0) {
-            position += bytesRead
-        }
-        return bytesRead
-    }
-
-    override fun write(src: ByteBuffer): Int {
-        throw UnsupportedOperationException("Read-only channel")
-    }
-
-    override fun position(): Long {
-        return position
-    }
-
-    override fun position(newPosition: Long): SeekableByteChannel {
-        throw UnsupportedOperationException("Seek not supported in streaming read")
-    }
-
-    override fun size(): Long {
-        return stat.size()
-    }
-
-    override fun truncate(size: Long): SeekableByteChannel {
-        throw UnsupportedOperationException("Read-only channel")
-    }
-
-    override fun isOpen(): Boolean {
-        return channel.isOpen
-    }
-
-    override fun close() {
-        IOUtils.closeQuietly(channel)
-    }
-}

plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3WriteSeekableByteChannel.kt

@@ -1,44 +0,0 @@
-package app.termora.plugins.s3
-
-import org.apache.commons.io.IOUtils
-import java.nio.ByteBuffer
-import java.nio.channels.SeekableByteChannel
-import java.nio.channels.WritableByteChannel
-
-class S3WriteSeekableByteChannel(
-    private val channel: WritableByteChannel,
-) : SeekableByteChannel {
-
-
-    override fun read(dst: ByteBuffer): Int {
-        throw UnsupportedOperationException("read not supported")
-    }
-
-    override fun write(src: ByteBuffer): Int {
-        return channel.write(src)
-    }
-
-    override fun position(): Long {
-        throw UnsupportedOperationException("position not supported")
-    }
-
-    override fun position(newPosition: Long): SeekableByteChannel {
-        throw UnsupportedOperationException("position not supported")
-    }
-
-    override fun size(): Long {
-        throw UnsupportedOperationException("size not supported")
-    }
-
-    override fun truncate(size: Long): SeekableByteChannel {
-        throw UnsupportedOperationException("truncate not supported")
-    }
-
-    override fun isOpen(): Boolean {
-        return channel.isOpen
-    }
-
-    override fun close() {
-        IOUtils.closeQuietly(channel)
-    }
-}

plugins/s3/src/test/kotlin/app/termora/plugins/s3/S3FileSystemTest.kt

@@ -58,7 +58,7 @@ class S3FileSystemTest {
             }
         }
 
-        val fileSystem = S3FileSystem(minioClient)
+        val fileSystem = MyS3FileSystem(minioClient)
         val path = fileSystem.getPath("/")
         PathWalker.walkFileTree(path, object : PathVisitor {
             override fun preVisitDirectory(