feat: support tencent cos

This commit is contained in:
hstyi
2025-06-26 18:05:39 +08:00
committed by GitHub
parent e25bd485ac
commit 007318dae3
36 changed files with 853 additions and 450 deletions

View File

@@ -0,0 +1,69 @@
package app.termora.plugins.cos
import app.termora.AuthenticationType
import app.termora.Proxy
import app.termora.ProxyType
import com.qcloud.cos.COSClient
import com.qcloud.cos.ClientConfig
import com.qcloud.cos.auth.BasicCOSCredentials
import com.qcloud.cos.model.Bucket
import com.qcloud.cos.region.Region
import java.io.Closeable
import java.util.concurrent.atomic.AtomicBoolean
class COSClientHandler(
private val cred: BasicCOSCredentials,
private val proxy: Proxy,
val buckets: List<Bucket>
) : Closeable {
companion object {
fun createCOSClient(cred: BasicCOSCredentials, region: String, proxy: Proxy): COSClient {
val clientConfig = ClientConfig()
if (region.isNotBlank()) {
clientConfig.region = Region(region)
}
clientConfig.isPrintShutdownStackTrace = false
if (proxy.type == ProxyType.HTTP) {
clientConfig.httpProxyIp = proxy.host
clientConfig.httpProxyPort = proxy.port
if (proxy.authenticationType == AuthenticationType.Password) {
clientConfig.proxyPassword = proxy.password
clientConfig.proxyUsername = proxy.username
}
}
return COSClient(cred, clientConfig)
}
}
/**
* key: Region
* value: Client
*/
private val clients = mutableMapOf<String, COSClient>()
private val closed = AtomicBoolean(false)
fun getClientForBucket(bucket: String): COSClient {
if (closed.get()) throw IllegalStateException("Client already closed")
synchronized(this) {
val bucket = buckets.first { it.name == bucket }
if (clients.containsKey(bucket.location)) {
return clients.getValue(bucket.location)
}
clients[bucket.location] = createCOSClient(cred, bucket.location, proxy)
return clients.getValue(bucket.location)
}
}
override fun close() {
if (closed.compareAndSet(false, true)) {
synchronized(this) {
clients.forEach { it.value.shutdown() }
clients.clear()
}
}
}
}

View File

@@ -1,41 +0,0 @@
package app.termora.plugins.cos
import org.apache.commons.vfs2.Capability
import org.apache.commons.vfs2.FileName
import org.apache.commons.vfs2.FileSystem
import org.apache.commons.vfs2.FileSystemOptions
import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider
class COSFileProvider private constructor() : AbstractOriginatingFileProvider() {
companion object {
val instance by lazy { COSFileProvider() }
val capabilities = listOf(
Capability.CREATE,
Capability.DELETE,
Capability.RENAME,
Capability.GET_TYPE,
Capability.LIST_CHILDREN,
Capability.READ_CONTENT,
Capability.URI,
Capability.WRITE_CONTENT,
Capability.GET_LAST_MODIFIED,
Capability.SET_LAST_MODIFIED_FILE,
Capability.RANDOM_ACCESS_READ,
Capability.APPEND_CONTENT
)
}
override fun getCapabilities(): Collection<Capability> {
return COSFileProvider.capabilities
}
override fun doCreateFileSystem(
rootFileName: FileName,
fileSystemOptions: FileSystemOptions
): FileSystem? {
TODO("Not yet implemented")
}
}

View File

@@ -0,0 +1,16 @@
package app.termora.plugins.cos
import app.termora.transfer.s3.S3FileSystem
import org.apache.commons.io.IOUtils
/**
* key: region
*/
class COSFileSystem(private val clientHandler: COSClientHandler) :
S3FileSystem(COSFileSystemProvider(clientHandler)) {
override fun close() {
IOUtils.closeQuietly(clientHandler)
super.close()
}
}

View File

@@ -0,0 +1,142 @@
package app.termora.plugins.cos
import app.termora.transfer.s3.S3FileAttributes
import app.termora.transfer.s3.S3FileSystemProvider
import app.termora.transfer.s3.S3Path
import com.qcloud.cos.model.ListObjectsRequest
import com.qcloud.cos.model.ObjectMetadata
import com.qcloud.cos.model.PutObjectRequest
import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.StringUtils
import java.io.InputStream
import java.io.OutputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.nio.file.AccessMode
import java.nio.file.NoSuchFileException
import java.util.concurrent.atomic.AtomicReference
import kotlin.io.path.absolutePathString
class COSFileSystemProvider(private val clientHandler: COSClientHandler) : S3FileSystemProvider() {
override fun getScheme(): String? {
return "s3"
}
override fun getOutputStream(path: S3Path): OutputStream {
return createStreamer(path)
}
override fun getInputStream(path: S3Path): InputStream {
val client = clientHandler.getClientForBucket(path.bucketName)
return client.getObject(path.bucketName, path.objectName).objectContent
}
private fun createStreamer(path: S3Path): OutputStream {
val pis = PipedInputStream()
val pos = PipedOutputStream(pis)
val exception = AtomicReference<Throwable>()
val thread = Thread.ofVirtual().start {
try {
val client = clientHandler.getClientForBucket(path.bucketName)
client.putObject(PutObjectRequest(path.bucketName, path.objectName, pis, ObjectMetadata()))
} catch (e: Exception) {
exception.set(e)
} finally {
IOUtils.closeQuietly(pis)
}
}
return object : OutputStream() {
override fun write(b: Int) {
val exception = exception.get()
if (exception != null) throw exception
pos.write(b)
}
override fun close() {
pos.close()
if (thread.isAlive) thread.join()
}
}
}
override fun fetchChildren(path: S3Path): MutableList<S3Path> {
val paths = mutableListOf<S3Path>()
// root
if (path.isRoot) {
for (bucket in clientHandler.buckets) {
val p = path.resolve(bucket.name)
p.attributes = S3FileAttributes(
directory = true,
lastModifiedTime = bucket.creationDate.toInstant().toEpochMilli()
)
paths.add(p)
}
return paths
}
var nextMarker = StringUtils.EMPTY
val maxKeys = 100
val bucketName = path.bucketName
while (true) {
val request = ListObjectsRequest()
.withBucketName(bucketName)
.withMaxKeys(maxKeys)
.withDelimiter(path.fileSystem.separator)
if (path.objectName.isNotBlank()) request.withPrefix(path.objectName + path.fileSystem.separator)
if (nextMarker.isNotBlank()) request.withMarker(nextMarker)
val objectListing = clientHandler.getClientForBucket(bucketName).listObjects(request)
for (e in objectListing.commonPrefixes) {
val p = path.bucket.resolve(e)
p.attributes = p.attributes.copy(directory = true)
delete(p)
paths.add(p)
}
for (e in objectListing.objectSummaries) {
val p = path.bucket.resolve(e.key)
p.attributes = p.attributes.copy(
regularFile = true, size = e.size,
lastModifiedTime = e.lastModified.time
)
paths.add(p)
}
if (objectListing.isTruncated.not()) {
break
}
nextMarker = objectListing.nextMarker
}
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
return paths
}
override fun delete(path: S3Path, isDirectory: Boolean) {
if (isDirectory.not())
clientHandler.getClientForBucket(path.bucketName).deleteObject(path.bucketName, path.objectName)
}
override fun checkAccess(path: S3Path, vararg modes: AccessMode) {
try {
val client = clientHandler.getClientForBucket(path.bucketName)
if (client.doesObjectExist(path.bucketName, path.objectName).not()) {
throw NoSuchFileException(path.objectName)
}
} catch (e: Exception) {
if (e is NoSuchFileException) throw e
throw NoSuchFileException(e.message)
}
}
}

View File

@@ -0,0 +1,313 @@
package app.termora.plugins.cos
import app.termora.*
import app.termora.plugin.internal.BasicProxyOption
import com.formdev.flatlaf.FlatClientProperties
import com.formdev.flatlaf.ui.FlatTextBorder
import com.jgoodies.forms.builder.FormBuilder
import com.jgoodies.forms.layout.FormLayout
import org.apache.commons.lang3.StringUtils
import java.awt.BorderLayout
import java.awt.KeyboardFocusManager
import java.awt.event.ComponentAdapter
import java.awt.event.ComponentEvent
import javax.swing.*
class COSHostOptionsPane : OptionsPane() {
private val generalOption = GeneralOption()
private val proxyOption = BasicProxyOption(listOf(ProxyType.HTTP))
private val sftpOption = SFTPOption()
init {
addOption(generalOption)
addOption(proxyOption)
addOption(sftpOption)
}
fun getHost(): Host {
val name = generalOption.nameTextField.text
val protocol = COSProtocolProvider.PROTOCOL
val port = 0
var authentication = Authentication.Companion.No
var proxy = Proxy.Companion.No
val authenticationType = AuthenticationType.Password
authentication = authentication.copy(
type = authenticationType,
password = String(generalOption.passwordTextField.password)
)
if (proxyOption.proxyTypeComboBox.selectedItem != ProxyType.No) {
proxy = proxy.copy(
type = proxyOption.proxyTypeComboBox.selectedItem as ProxyType,
host = proxyOption.proxyHostTextField.text,
username = proxyOption.proxyUsernameTextField.text,
password = String(proxyOption.proxyPasswordTextField.password),
port = proxyOption.proxyPortTextField.value as Int,
authenticationType = proxyOption.proxyAuthenticationTypeComboBox.selectedItem as AuthenticationType,
)
}
val options = Options.Default.copy(
sftpDefaultDirectory = sftpOption.defaultDirectoryField.text,
extras = mutableMapOf(
"cos.delimiter" to generalOption.delimiterTextField.text,
)
)
return Host(
name = name,
protocol = protocol,
port = port,
username = generalOption.usernameTextField.text,
authentication = authentication,
proxy = proxy,
sort = System.currentTimeMillis(),
remark = generalOption.remarkTextArea.text,
options = options,
)
}
fun setHost(host: Host) {
generalOption.nameTextField.text = host.name
generalOption.usernameTextField.text = host.username
generalOption.remarkTextArea.text = host.remark
generalOption.passwordTextField.text = host.authentication.password
generalOption.delimiterTextField.text = host.options.extras["cos.delimiter"] ?: StringUtils.EMPTY
proxyOption.proxyTypeComboBox.selectedItem = host.proxy.type
proxyOption.proxyHostTextField.text = host.proxy.host
proxyOption.proxyPasswordTextField.text = host.proxy.password
proxyOption.proxyUsernameTextField.text = host.proxy.username
proxyOption.proxyPortTextField.value = host.proxy.port
proxyOption.proxyAuthenticationTypeComboBox.selectedItem = host.proxy.authenticationType
sftpOption.defaultDirectoryField.text = host.options.sftpDefaultDirectory
}
fun validateFields(): Boolean {
val host = getHost()
// general
if (validateField(generalOption.nameTextField)) {
return false
}
if (validateField(generalOption.usernameTextField)) {
return false
}
if (host.authentication.type == AuthenticationType.Password) {
if (validateField(generalOption.passwordTextField)) {
return false
}
}
// proxy
if (host.proxy.type != ProxyType.No) {
if (validateField(proxyOption.proxyHostTextField)
) {
return false
}
if (host.proxy.authenticationType != AuthenticationType.No) {
if (validateField(proxyOption.proxyUsernameTextField)
|| validateField(proxyOption.proxyPasswordTextField)
) {
return false
}
}
}
return true
}
/**
* 返回 true 表示有错误
*/
private fun validateField(textField: JTextField): Boolean {
if (textField.isEnabled && textField.text.isBlank()) {
setOutlineError(textField)
return true
}
return false
}
private fun setOutlineError(c: JComponent) {
selectOptionJComponent(c)
c.putClientProperty(FlatClientProperties.OUTLINE, FlatClientProperties.OUTLINE_ERROR)
c.requestFocusInWindow()
}
private inner class GeneralOption : JPanel(BorderLayout()), Option {
val nameTextField = OutlineTextField(128)
val usernameTextField = OutlineTextField(128)
val passwordTextField = OutlinePasswordField(255)
val remarkTextArea = FixedLengthTextArea(512)
// val regionComboBox = OutlineComboBox<String>()
val delimiterTextField = OutlineTextField(128)
init {
initView()
initEvents()
}
private fun initView() {
/*regionComboBox.addItem("ap-beijing-1")
regionComboBox.addItem("ap-beijing")
regionComboBox.addItem("ap-nanjing")
regionComboBox.addItem("ap-shanghai")
regionComboBox.addItem("ap-guangzhou")
regionComboBox.addItem("ap-chengdu")
regionComboBox.addItem("ap-chongqing")
regionComboBox.addItem("ap-shenzhen-fsi")
regionComboBox.addItem("ap-shanghai-fsi")
regionComboBox.addItem("ap-beijing-fsi")
regionComboBox.addItem("ap-hongkong")
regionComboBox.addItem("ap-singapore")
regionComboBox.addItem("ap-jakarta")
regionComboBox.addItem("ap-seoul")
regionComboBox.addItem("ap-bangkok")
regionComboBox.addItem("ap-tokyo")
regionComboBox.addItem("na-siliconvalley")
regionComboBox.addItem("na-ashburn")
regionComboBox.addItem("sa-saopaulo")
regionComboBox.addItem("eu-frankfurt")
regionComboBox.isEditable = true*/
delimiterTextField.text = "/"
delimiterTextField.isEditable = false
add(getCenterComponent(), BorderLayout.CENTER)
}
private fun initEvents() {
addComponentListener(object : ComponentAdapter() {
override fun componentResized(e: ComponentEvent) {
SwingUtilities.invokeLater { nameTextField.requestFocusInWindow() }
removeComponentListener(this)
}
})
}
override fun getIcon(isSelected: Boolean): Icon {
return Icons.settings
}
override fun getTitle(): String {
return I18n.getString("termora.new-host.general")
}
override fun getJComponent(): JComponent {
return this
}
private fun getCenterComponent(): JComponent {
val layout = FormLayout(
"left:pref, $FORM_MARGIN, default:grow, $FORM_MARGIN, pref, $FORM_MARGIN, default",
"pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref"
)
remarkTextArea.setFocusTraversalKeys(
KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS,
KeyboardFocusManager.getCurrentKeyboardFocusManager()
.getDefaultFocusTraversalKeys(KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS)
)
remarkTextArea.setFocusTraversalKeys(
KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS,
KeyboardFocusManager.getCurrentKeyboardFocusManager()
.getDefaultFocusTraversalKeys(KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS)
)
remarkTextArea.rows = 8
remarkTextArea.lineWrap = true
remarkTextArea.border = BorderFactory.createEmptyBorder(4, 4, 4, 4)
var rows = 1
val step = 2
val panel = FormBuilder.create().layout(layout)
.add("${I18n.getString("termora.new-host.general.name")}:").xy(1, rows)
.add(nameTextField).xyw(3, rows, 5).apply { rows += step }
.add("SecretId:").xy(1, rows)
.add(usernameTextField).xyw(3, rows, 5).apply { rows += step }
.add("SecretKey:").xy(1, rows)
.add(passwordTextField).xyw(3, rows, 5).apply { rows += step }
.add("Delimiter:").xy(1, rows)
.add(delimiterTextField).xyw(3, rows, 5).apply { rows += step }
.add("${I18n.getString("termora.new-host.general.remark")}:").xy(1, rows)
.add(JScrollPane(remarkTextArea).apply { border = FlatTextBorder() })
.xyw(3, rows, 5).apply { rows += step }
.build()
return panel
}
}
private inner class SFTPOption : JPanel(BorderLayout()), Option {
val defaultDirectoryField = OutlineTextField(255)
init {
initView()
initEvents()
}
private fun initView() {
add(getCenterComponent(), BorderLayout.CENTER)
}
private fun initEvents() {
}
override fun getIcon(isSelected: Boolean): Icon {
return Icons.folder
}
override fun getTitle(): String {
return I18n.getString("termora.transport.sftp")
}
override fun getJComponent(): JComponent {
return this
}
private fun getCenterComponent(): JComponent {
val layout = FormLayout(
"left:pref, $FORM_MARGIN, default:grow",
"pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref"
)
var rows = 1
val step = 2
val panel = FormBuilder.create().layout(layout)
.add("${I18n.getString("termora.settings.sftp.default-directory")}:").xy(1, rows)
.add(defaultDirectoryField).xy(3, rows).apply { rows += step }
.build()
return panel
}
}
}

View File

@@ -1,8 +1,5 @@
package app.termora.plugins.cos
import app.termora.DynamicIcon
import app.termora.I18n
import app.termora.Icons
import app.termora.plugin.Extension
import app.termora.plugin.ExtensionSupport
import app.termora.plugin.PaidPlugin
@@ -13,8 +10,8 @@ class COSPlugin : PaidPlugin {
private val support = ExtensionSupport()
init {
support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.Companion.instance }
support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.Companion.instance }
support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.instance }
support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.instance }
}
override fun getAuthor(): String {

View File

@@ -1,22 +1,36 @@
package app.termora.plugins.cos
import app.termora.Disposer
import app.termora.Host
import app.termora.protocol.ProtocolHostPanel
import org.apache.commons.lang3.StringUtils
import java.awt.BorderLayout
class COSProtocolHostPanel : ProtocolHostPanel() {
private val pane = COSHostOptionsPane()
init {
initView()
initEvents()
}
private fun initView() {
add(pane, BorderLayout.CENTER)
Disposer.register(this, pane)
}
private fun initEvents() {}
override fun getHost(): Host {
return Host(
name = StringUtils.EMPTY,
protocol = COSProtocolProvider.Companion.PROTOCOL
)
return pane.getHost()
}
override fun setHost(host: Host) {
pane.setHost(host)
}
override fun validateFields(): Boolean {
return true
return pane.validateFields()
}
}

View File

@@ -10,7 +10,7 @@ class COSProtocolHostPanelExtension private constructor() : ProtocolHostPanelExt
}
override fun getProtocolProvider(): ProtocolProvider {
return COSProtocolProvider.Companion.instance
return COSProtocolProvider.instance
}
override fun createProtocolHostPanel(): ProtocolHostPanel {

View File

@@ -2,10 +2,14 @@ package app.termora.plugins.cos
import app.termora.DynamicIcon
import app.termora.Icons
import app.termora.protocol.FileObjectHandler
import app.termora.protocol.FileObjectRequest
import app.termora.protocol.PathHandler
import app.termora.protocol.PathHandlerRequest
import app.termora.protocol.TransferProtocolProvider
import org.apache.commons.vfs2.provider.FileProvider
import com.qcloud.cos.ClientConfig
import com.qcloud.cos.auth.BasicCOSCredentials
import com.qcloud.cos.model.Bucket
import org.apache.commons.lang3.StringUtils
class COSProtocolProvider private constructor() : TransferProtocolProvider {
@@ -22,12 +26,30 @@ class COSProtocolProvider private constructor() : TransferProtocolProvider {
return Icons.tencent
}
override fun getFileProvider(): FileProvider {
return COSFileProvider.instance
override fun createPathHandler(requester: PathHandlerRequest): PathHandler {
val host = requester.host
val secretId = host.username
val secretKey = host.authentication.password
val cred = BasicCOSCredentials(secretId, secretKey)
val clientConfig = ClientConfig()
clientConfig.isPrintShutdownStackTrace = false
val cosClient = COSClientHandler.createCOSClient(cred, StringUtils.EMPTY, host.proxy)
val buckets: List<Bucket>
try {
buckets = cosClient.listBuckets()
if (buckets.isEmpty()) {
throw IllegalStateException("没有获取到桶信息")
}
} finally {
cosClient.shutdown()
}
val defaultPath = host.options.sftpDefaultDirectory
val fs = COSFileSystem(COSClientHandler(cred, host.proxy, buckets))
return PathHandler(fs, fs.getPath(defaultPath))
}
override fun getRootFileObject(requester: FileObjectRequest): FileObjectHandler {
TODO("Not yet implemented")
}
}

View File

@@ -9,6 +9,6 @@ class COSProtocolProviderExtension private constructor() : ProtocolProviderExten
}
override fun getProtocolProvider(): ProtocolProvider {
return COSProtocolProvider.Companion.instance
return COSProtocolProvider.instance
}
}

View File

@@ -3,7 +3,7 @@ plugins {
}
project.version = "0.0.4"
project.version = "0.0.5"
dependencies {

View File

@@ -0,0 +1,13 @@
package app.termora.plugins.s3
import app.termora.transfer.s3.S3FileSystem
import io.minio.MinioClient
class MyS3FileSystem(private val minioClient: MinioClient) :
S3FileSystem(MyS3FileSystemProvider(minioClient)) {
override fun close() {
minioClient.close()
super.close()
}
}

View File

@@ -0,0 +1,157 @@
package app.termora.plugins.s3
import app.termora.transfer.s3.S3FileAttributes
import app.termora.transfer.s3.S3FileSystemProvider
import app.termora.transfer.s3.S3Path
import io.minio.*
import io.minio.errors.ErrorResponseException
import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.StringUtils
import java.io.InputStream
import java.io.OutputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.nio.file.AccessMode
import java.nio.file.NoSuchFileException
import java.util.concurrent.atomic.AtomicReference
import kotlin.io.path.absolutePathString
class MyS3FileSystemProvider(private val minioClient: MinioClient) : S3FileSystemProvider() {
override fun getScheme(): String? {
return "s3"
}
override fun getOutputStream(path: S3Path): OutputStream {
return createStreamer(path)
}
override fun getInputStream(path: S3Path): InputStream {
return minioClient.getObject(
GetObjectArgs.builder().bucket(path.bucketName)
.`object`(path.objectName).build()
)
}
private fun createStreamer(path: S3Path): OutputStream {
val pis = PipedInputStream()
val pos = PipedOutputStream(pis)
val exception = AtomicReference<Throwable>()
val thread = Thread.ofVirtual().start {
try {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(path.bucketName)
.stream(pis, -1, 32 * 1024 * 1024)
.`object`(path.objectName).build()
)
} catch (e: Exception) {
exception.set(e)
} finally {
IOUtils.closeQuietly(pis)
}
}
return object : OutputStream() {
override fun write(b: Int) {
val exception = exception.get()
if (exception != null) throw exception
pos.write(b)
}
override fun close() {
pos.close()
if (thread.isAlive) thread.join()
}
}
}
override fun fetchChildren(path: S3Path): MutableList<S3Path> {
val paths = mutableListOf<S3Path>()
// root
if (path.isRoot) {
for (bucket in minioClient.listBuckets()) {
val p = path.resolve(bucket.name())
p.attributes = S3FileAttributes(
directory = true,
lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
)
paths.add(p)
}
return paths
}
var startAfter = StringUtils.EMPTY
val maxKeys = 100
while (true) {
val builder = ListObjectsArgs.builder()
.bucket(path.bucketName)
.maxKeys(maxKeys)
.delimiter(path.fileSystem.separator)
if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
if (startAfter.isNotBlank()) builder.startAfter(startAfter)
val subPaths = mutableListOf<S3Path>()
for (e in minioClient.listObjects(builder.build())) {
val item = e.get()
val p = path.bucket.resolve(item.objectName())
var attributes = p.attributes.copy(
directory = item.isDir,
regularFile = item.isDir.not(),
size = item.size()
)
if (item.lastModified() != null) {
attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
}
p.attributes = attributes
// 如果是文件夹,那么就要删除内存中的
if (attributes.isDirectory) {
delete(p)
}
subPaths.add(p)
startAfter = item.objectName()
}
paths.addAll(subPaths)
if (subPaths.size < maxKeys)
break
}
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
return paths
}
override fun delete(path: S3Path, isDirectory: Boolean) {
if (isDirectory.not())
minioClient.removeObject(
RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
)
}
override fun checkAccess(path: S3Path, vararg modes: AccessMode) {
try {
minioClient.statObject(
StatObjectArgs.builder()
.`object`(path.objectName)
.bucket(path.bucketName).build()
)
} catch (e: ErrorResponseException) {
throw NoSuchFileException(e.errorResponse().message())
}
}
}

View File

@@ -1,52 +0,0 @@
package app.termora.plugins.s3
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
data class S3FileAttributes(
private val lastModifiedTime: Long = 0,
private val lastAccessTime: Long = 0,
private val creationTime: Long = 0,
private val regularFile: Boolean = false,
private val directory: Boolean = false,
private val symbolicLink: Boolean = false,
private val other: Boolean = false,
private val size: Long = 0,
) : BasicFileAttributes {
override fun lastModifiedTime(): FileTime {
return FileTime.fromMillis(lastModifiedTime)
}
override fun lastAccessTime(): FileTime {
return FileTime.fromMillis(lastAccessTime)
}
override fun creationTime(): FileTime {
return FileTime.fromMillis(creationTime)
}
override fun isRegularFile(): Boolean {
return regularFile
}
override fun isDirectory(): Boolean {
return directory
}
override fun isSymbolicLink(): Boolean {
return symbolicLink
}
override fun isOther(): Boolean {
return other
}
override fun size(): Long {
return size
}
override fun fileKey(): Any? {
return null
}
}

View File

@@ -1,44 +0,0 @@
package app.termora.plugins.s3
import io.minio.MinioClient
import org.apache.sshd.common.file.util.BaseFileSystem
import java.nio.file.Path
import java.nio.file.attribute.UserPrincipalLookupService
import java.util.concurrent.atomic.AtomicBoolean
class S3FileSystem(
private val minioClient: MinioClient,
) : BaseFileSystem<S3Path>(S3FileSystemProvider(minioClient)) {
private val isOpen = AtomicBoolean(true)
override fun create(root: String?, names: List<String>): S3Path {
val path = S3Path(this, root, names)
if (names.isEmpty()) {
path.attributes = path.attributes.copy(directory = true)
}
return path
}
override fun close() {
if (isOpen.compareAndSet(false, true)) {
minioClient.close()
}
}
override fun isOpen(): Boolean {
return isOpen.get()
}
override fun getRootDirectories(): Iterable<Path> {
return mutableSetOf<Path>(create(separator))
}
override fun supportedFileAttributeViews(): Set<String> {
TODO("Not yet implemented")
}
override fun getUserPrincipalLookupService(): UserPrincipalLookupService {
throw UnsupportedOperationException()
}
}

View File

@@ -1,308 +0,0 @@
package app.termora.plugins.s3
import io.minio.*
import io.minio.errors.ErrorResponseException
import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.StringUtils
import java.io.OutputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.net.URI
import java.nio.channels.Channels
import java.nio.channels.SeekableByteChannel
import java.nio.file.*
import java.nio.file.attribute.*
import java.nio.file.spi.FileSystemProvider
import java.util.concurrent.atomic.AtomicReference
import kotlin.io.path.absolutePathString
import kotlin.io.path.name
class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemProvider() {
/**
* 因为 S3 协议不存在文件夹,所以用户新建的文件夹先保存到内存中
*/
private val directories = mutableMapOf<String, MutableList<S3Path>>()
override fun getScheme(): String? {
return "s3"
}
override fun newFileSystem(
uri: URI,
env: Map<String, *>
): FileSystem {
TODO("Not yet implemented")
}
override fun getFileSystem(uri: URI): FileSystem {
TODO("Not yet implemented")
}
override fun getPath(uri: URI): Path {
TODO("Not yet implemented")
}
override fun newByteChannel(
path: Path,
options: Set<OpenOption>,
vararg attrs: FileAttribute<*>
): SeekableByteChannel {
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
if (options.contains(StandardOpenOption.WRITE)) {
return S3WriteSeekableByteChannel(Channels.newChannel(createStreamer(path)))
} else {
val response = minioClient.getObject(
GetObjectArgs.builder().bucket(path.bucketName)
.`object`(path.objectName).build()
)
return S3ReadSeekableByteChannel(Channels.newChannel(response), stat(path))
}
}
private fun createStreamer(path: S3Path): OutputStream {
val pis = PipedInputStream()
val pos = PipedOutputStream(pis)
val exception = AtomicReference<Throwable>()
val thread = Thread.ofVirtual().start {
try {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(path.bucketName)
.stream(pis, -1, 32 * 1024 * 1024)
.`object`(path.objectName).build()
)
} catch (e: Exception) {
exception.set(e)
} finally {
IOUtils.closeQuietly(pis)
}
}
return object : OutputStream() {
override fun write(b: Int) {
val exception = exception.get()
if (exception != null) throw exception
pos.write(b)
}
override fun close() {
pos.close()
if (thread.isAlive) thread.join()
}
}
}
override fun newDirectoryStream(
dir: Path,
filter: DirectoryStream.Filter<in Path>
): DirectoryStream<Path> {
return object : DirectoryStream<Path> {
override fun iterator(): MutableIterator<Path> {
return files(dir as S3Path).iterator()
}
override fun close() {
}
}
}
private fun files(path: S3Path): MutableList<S3Path> {
val paths = mutableListOf<S3Path>()
// root
if (path.isRoot) {
for (bucket in minioClient.listBuckets()) {
val p = path.resolve(bucket.name())
p.attributes = S3FileAttributes(
directory = true,
lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
)
paths.add(p)
}
return paths
}
var startAfter = StringUtils.EMPTY
val maxKeys = 100
while (true) {
val builder = ListObjectsArgs.builder()
.bucket(path.bucketName)
.maxKeys(maxKeys)
.delimiter(path.fileSystem.separator)
if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
if (startAfter.isNotBlank()) builder.startAfter(startAfter)
val subPaths = mutableListOf<S3Path>()
for (e in minioClient.listObjects(builder.build())) {
val item = e.get()
val p = path.bucket.resolve(item.objectName())
var attributes = p.attributes.copy(
directory = item.isDir,
regularFile = item.isDir.not(),
size = item.size()
)
if (item.lastModified() != null) {
attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
}
p.attributes = attributes
// 如果是文件夹,那么就要删除内存中的
if (attributes.isDirectory) {
delete(p)
}
subPaths.add(p)
startAfter = item.objectName()
}
paths.addAll(subPaths)
if (subPaths.size < maxKeys)
break
}
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
return paths
}
override fun createDirectory(dir: Path, vararg attrs: FileAttribute<*>) {
synchronized(this) {
if (dir !is S3Path) throw UnsupportedOperationException("dir must be a S3Path")
if (dir.isRoot || dir.isBucket) throw UnsupportedOperationException("No operation permission")
val parent = dir.parent ?: throw UnsupportedOperationException("No operation permission")
directories.computeIfAbsent(parent.absolutePathString()) { mutableListOf() }
.add(dir.apply {
attributes = attributes.copy(directory = true, lastModifiedTime = System.currentTimeMillis())
})
}
}
override fun delete(path: Path) {
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
if (path.attributes.isDirectory) {
val parent = path.parent
if (parent != null) {
synchronized(this) {
directories[parent.absolutePathString()]?.removeIf { it.name == path.name }
}
}
return
}
minioClient.removeObject(
RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
)
}
override fun copy(
source: Path?,
target: Path?,
vararg options: CopyOption?
) {
throw UnsupportedOperationException()
}
override fun move(
source: Path?,
target: Path?,
vararg options: CopyOption?
) {
throw UnsupportedOperationException()
}
override fun isSameFile(path: Path?, path2: Path?): Boolean {
throw UnsupportedOperationException()
}
override fun isHidden(path: Path?): Boolean {
throw UnsupportedOperationException()
}
override fun getFileStore(path: Path?): FileStore? {
throw UnsupportedOperationException()
}
override fun checkAccess(path: Path, vararg modes: AccessMode) {
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
try {
stat(path)
} catch (e: ErrorResponseException) {
throw NoSuchFileException(e.errorResponse().message())
}
}
private fun stat(path: S3Path): StatObjectResponse {
return minioClient.statObject(
StatObjectArgs.builder()
.`object`(path.objectName)
.bucket(path.bucketName).build()
)
}
override fun <V : FileAttributeView> getFileAttributeView(
path: Path,
type: Class<V>,
vararg options: LinkOption?
): V {
if (path is S3Path) {
return type.cast(object : BasicFileAttributeView {
override fun name(): String {
return "basic"
}
override fun readAttributes(): BasicFileAttributes {
return path.attributes
}
override fun setTimes(
lastModifiedTime: FileTime?,
lastAccessTime: FileTime?,
createTime: FileTime?
) {
throw UnsupportedOperationException()
}
})
}
throw UnsupportedOperationException()
}
override fun <A : BasicFileAttributes> readAttributes(
path: Path,
type: Class<A>,
vararg options: LinkOption
): A {
if (path is S3Path) {
return type.cast(getFileAttributeView(path, BasicFileAttributeView::class.java).readAttributes())
}
throw UnsupportedOperationException()
}
override fun readAttributes(
path: Path?,
attributes: String?,
vararg options: LinkOption?
): Map<String?, Any?>? {
throw UnsupportedOperationException()
}
override fun setAttribute(
path: Path?,
attribute: String?,
value: Any?,
vararg options: LinkOption?
) {
throw UnsupportedOperationException()
}
}

View File

@@ -1,60 +0,0 @@
package app.termora.plugins.s3
import app.termora.transfer.WithPathAttributes
import org.apache.sshd.common.file.util.BasePath
import java.nio.file.LinkOption
import java.nio.file.Path
import kotlin.io.path.absolutePathString
class S3Path(
fileSystem: S3FileSystem,
root: String?,
names: List<String>,
) : BasePath<S3Path, S3FileSystem>(fileSystem, root, names), WithPathAttributes {
private val separator get() = fileSystem.separator
var attributes = S3FileAttributes()
/**
* 是否是 Bucket
*/
val isBucket get() = parent != null && parent?.parent == null
/**
* 是否是根
*/
val isRoot get() = absolutePathString() == separator
/**
* Bucket Name
*/
val bucketName: String get() = names.first()
/**
* 获取 Bucket
*/
val bucket: S3Path get() = fileSystem.getPath(root, bucketName)
/**
* 获取所在 Bucket 的路径
*/
val objectName: String get() = names.subList(1, names.size).joinToString(separator)
override fun getCustomType(): String? {
if (isBucket) return "Bucket"
return null
}
override fun toRealPath(vararg options: LinkOption): Path {
return toAbsolutePath()
}
override fun getParent(): S3Path? {
val path = super.getParent() ?: return null
path.attributes = path.attributes.copy(directory = true)
return path
}
}

View File

@@ -70,7 +70,7 @@ class S3ProtocolProvider private constructor() : TransferProtocolProvider {
// val delimiter = host.options.extras["s3.delimiter"] ?: "/"
val defaultPath = host.options.sftpDefaultDirectory
val minioClient = builder.build()
val fs = S3FileSystem(minioClient)
val fs = MyS3FileSystem(minioClient)
return PathHandler(fs, fs.getPath(defaultPath))
}

View File

@@ -1,51 +0,0 @@
package app.termora.plugins.s3
import io.minio.StatObjectResponse
import org.apache.commons.io.IOUtils
import java.nio.ByteBuffer
import java.nio.channels.ReadableByteChannel
import java.nio.channels.SeekableByteChannel
class S3ReadSeekableByteChannel(
private val channel: ReadableByteChannel,
private val stat: StatObjectResponse
) : SeekableByteChannel {
private var position: Long = 0
override fun read(dst: ByteBuffer): Int {
val bytesRead = channel.read(dst)
if (bytesRead > 0) {
position += bytesRead
}
return bytesRead
}
override fun write(src: ByteBuffer): Int {
throw UnsupportedOperationException("Read-only channel")
}
override fun position(): Long {
return position
}
override fun position(newPosition: Long): SeekableByteChannel {
throw UnsupportedOperationException("Seek not supported in streaming read")
}
override fun size(): Long {
return stat.size()
}
override fun truncate(size: Long): SeekableByteChannel {
throw UnsupportedOperationException("Read-only channel")
}
override fun isOpen(): Boolean {
return channel.isOpen
}
override fun close() {
IOUtils.closeQuietly(channel)
}
}

View File

@@ -1,44 +0,0 @@
package app.termora.plugins.s3
import org.apache.commons.io.IOUtils
import java.nio.ByteBuffer
import java.nio.channels.SeekableByteChannel
import java.nio.channels.WritableByteChannel
class S3WriteSeekableByteChannel(
private val channel: WritableByteChannel,
) : SeekableByteChannel {
override fun read(dst: ByteBuffer): Int {
throw UnsupportedOperationException("read not supported")
}
override fun write(src: ByteBuffer): Int {
return channel.write(src)
}
override fun position(): Long {
throw UnsupportedOperationException("position not supported")
}
override fun position(newPosition: Long): SeekableByteChannel {
throw UnsupportedOperationException("position not supported")
}
override fun size(): Long {
throw UnsupportedOperationException("size not supported")
}
override fun truncate(size: Long): SeekableByteChannel {
throw UnsupportedOperationException("truncate not supported")
}
override fun isOpen(): Boolean {
return channel.isOpen
}
override fun close() {
IOUtils.closeQuietly(channel)
}
}

View File

@@ -58,7 +58,7 @@ class S3FileSystemTest {
}
}
val fileSystem = S3FileSystem(minioClient)
val fileSystem = MyS3FileSystem(minioClient)
val path = fileSystem.getPath("/")
PathWalker.walkFileTree(path, object : PathVisitor {
override fun preVisitDirectory(