From 007318dae3bd13f3bd0a67a5cc7079e5d13b3b33 Mon Sep 17 00:00:00 2001 From: hstyi Date: Thu, 26 Jun 2025 18:05:39 +0800 Subject: [PATCH] feat: support tencent cos --- .github/workflows/linux-aarch64.yml | 1 + .../termora/plugins/cos/COSClientHandler.kt | 69 ++++ .../termora/plugins/cos/COSFileProvider.kt | 41 --- .../app/termora/plugins/cos/COSFileSystem.kt | 16 + .../plugins/cos/COSFileSystemProvider.kt | 142 ++++++++ .../termora/plugins/cos/COSHostOptionsPane.kt | 313 ++++++++++++++++++ .../app/termora/plugins/cos/COSPlugin.kt | 7 +- .../plugins/cos/COSProtocolHostPanel.kt | 28 +- .../cos/COSProtocolHostPanelExtension.kt | 2 +- .../plugins/cos/COSProtocolProvider.kt | 38 ++- .../cos/COSProtocolProviderExtension.kt | 2 +- plugins/s3/build.gradle.kts | 2 +- .../app/termora/plugins/s3/MyS3FileSystem.kt | 13 + .../plugins/s3/MyS3FileSystemProvider.kt | 157 +++++++++ .../termora/plugins/s3/S3ProtocolProvider.kt | 2 +- .../termora/plugins/s3/S3FileSystemTest.kt | 2 +- settings.gradle.kts | 2 +- .../plugin/internal/BasicProxyOption.kt | 8 +- .../app/termora/protocol/PathHandler.kt | 5 +- .../terminal/panel/vw/TransferVisualWindow.kt | 6 - .../DefaultInternalTransferManager.kt | 21 +- .../app/termora/transfer/FileTransfer.kt | 17 +- .../termora/transfer/TransferTableModel.kt | 5 + .../app/termora/transfer/TransportPanel.kt | 2 +- .../app/termora/transfer/TransportViewer.kt | 6 - .../transfer/internal/sftp/SFTPPathHandler.kt | 2 +- .../termora/transfer}/s3/S3FileAttributes.kt | 2 +- .../app/termora/transfer}/s3/S3FileSystem.kt | 17 +- .../transfer}/s3/S3FileSystemProvider.kt | 165 ++------- .../kotlin/app/termora/transfer}/s3/S3Path.kt | 14 +- .../transfer}/s3/S3ReadSeekableByteChannel.kt | 15 +- .../s3/S3WriteSeekableByteChannel.kt | 11 +- src/test/kotlin/app/termora/ExposedTest.kt | 39 --- src/test/kotlin/app/termora/HostTest.kt | 2 +- .../app/termora/account/ServerManagerTest.kt | 15 - .../vfs2/sftp/MySftpFileProviderTest.kt | 114 ------- 36 files changed, 853 insertions(+), 450 deletions(-) create mode 100644 plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSClientHandler.kt delete mode 100644 plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileProvider.kt create mode 100644 plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystem.kt create mode 100644 plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystemProvider.kt create mode 100644 plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSHostOptionsPane.kt create mode 100644 plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystem.kt create mode 100644 plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystemProvider.kt rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3FileAttributes.kt (97%) rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3FileSystem.kt (77%) rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3FileSystemProvider.kt (50%) rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3Path.kt (72%) rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3ReadSeekableByteChannel.kt (77%) rename {plugins/s3/src/main/kotlin/app/termora/plugins => src/main/kotlin/app/termora/transfer}/s3/S3WriteSeekableByteChannel.kt (80%) delete mode 100644 src/test/kotlin/app/termora/ExposedTest.kt delete mode 100644 src/test/kotlin/app/termora/account/ServerManagerTest.kt delete mode 100644 src/test/kotlin/app/termora/vfs2/sftp/MySftpFileProviderTest.kt diff --git a/.github/workflows/linux-aarch64.yml b/.github/workflows/linux-aarch64.yml index e34dd8b..071d0a4 100644 --- a/.github/workflows/linux-aarch64.yml +++ b/.github/workflows/linux-aarch64.yml @@ -37,6 +37,7 @@ jobs: # test build - run: | ./gradlew build -x test --no-daemon + ./gradlew clean --no-daemon # dist - run: | diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSClientHandler.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSClientHandler.kt new file mode 100644 index 0000000..1c8c739 --- /dev/null +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSClientHandler.kt @@ -0,0 +1,69 @@ +package app.termora.plugins.cos + +import app.termora.AuthenticationType +import app.termora.Proxy +import app.termora.ProxyType +import com.qcloud.cos.COSClient +import com.qcloud.cos.ClientConfig +import com.qcloud.cos.auth.BasicCOSCredentials +import com.qcloud.cos.model.Bucket +import com.qcloud.cos.region.Region +import java.io.Closeable +import java.util.concurrent.atomic.AtomicBoolean + +class COSClientHandler( + private val cred: BasicCOSCredentials, + private val proxy: Proxy, + val buckets: List +) : Closeable { + + companion object { + fun createCOSClient(cred: BasicCOSCredentials, region: String, proxy: Proxy): COSClient { + val clientConfig = ClientConfig() + if (region.isNotBlank()) { + clientConfig.region = Region(region) + } + clientConfig.isPrintShutdownStackTrace = false + if (proxy.type == ProxyType.HTTP) { + clientConfig.httpProxyIp = proxy.host + clientConfig.httpProxyPort = proxy.port + if (proxy.authenticationType == AuthenticationType.Password) { + clientConfig.proxyPassword = proxy.password + clientConfig.proxyUsername = proxy.username + } + } + return COSClient(cred, clientConfig) + } + } + + /** + * key: Region + * value: Client + */ + private val clients = mutableMapOf() + private val closed = AtomicBoolean(false) + + fun getClientForBucket(bucket: String): COSClient { + if (closed.get()) throw IllegalStateException("Client already closed") + + synchronized(this) { + val bucket = buckets.first { it.name == bucket } + if (clients.containsKey(bucket.location)) { + return clients.getValue(bucket.location) + } + clients[bucket.location] = createCOSClient(cred, bucket.location, proxy) + return clients.getValue(bucket.location) + } + } + + override fun close() { + if (closed.compareAndSet(false, true)) { + synchronized(this) { + clients.forEach { it.value.shutdown() } + clients.clear() + } + } + } + + +} \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileProvider.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileProvider.kt deleted file mode 100644 index 7817e72..0000000 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileProvider.kt +++ /dev/null @@ -1,41 +0,0 @@ -package app.termora.plugins.cos - -import org.apache.commons.vfs2.Capability -import org.apache.commons.vfs2.FileName -import org.apache.commons.vfs2.FileSystem -import org.apache.commons.vfs2.FileSystemOptions -import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider - -class COSFileProvider private constructor() : AbstractOriginatingFileProvider() { - - companion object { - val instance by lazy { COSFileProvider() } - val capabilities = listOf( - Capability.CREATE, - Capability.DELETE, - Capability.RENAME, - Capability.GET_TYPE, - Capability.LIST_CHILDREN, - Capability.READ_CONTENT, - Capability.URI, - Capability.WRITE_CONTENT, - Capability.GET_LAST_MODIFIED, - Capability.SET_LAST_MODIFIED_FILE, - Capability.RANDOM_ACCESS_READ, - Capability.APPEND_CONTENT - ) - } - - override fun getCapabilities(): Collection { - return COSFileProvider.capabilities - } - - override fun doCreateFileSystem( - rootFileName: FileName, - fileSystemOptions: FileSystemOptions - ): FileSystem? { - TODO("Not yet implemented") - } - - -} \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystem.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystem.kt new file mode 100644 index 0000000..6fb275a --- /dev/null +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystem.kt @@ -0,0 +1,16 @@ +package app.termora.plugins.cos + +import app.termora.transfer.s3.S3FileSystem +import org.apache.commons.io.IOUtils + +/** + * key: region + */ +class COSFileSystem(private val clientHandler: COSClientHandler) : + S3FileSystem(COSFileSystemProvider(clientHandler)) { + + override fun close() { + IOUtils.closeQuietly(clientHandler) + super.close() + } +} diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystemProvider.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystemProvider.kt new file mode 100644 index 0000000..fbffe1f --- /dev/null +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSFileSystemProvider.kt @@ -0,0 +1,142 @@ +package app.termora.plugins.cos + +import app.termora.transfer.s3.S3FileAttributes +import app.termora.transfer.s3.S3FileSystemProvider +import app.termora.transfer.s3.S3Path +import com.qcloud.cos.model.ListObjectsRequest +import com.qcloud.cos.model.ObjectMetadata +import com.qcloud.cos.model.PutObjectRequest +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.StringUtils +import java.io.InputStream +import java.io.OutputStream +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.nio.file.AccessMode +import java.nio.file.NoSuchFileException +import java.util.concurrent.atomic.AtomicReference +import kotlin.io.path.absolutePathString + +class COSFileSystemProvider(private val clientHandler: COSClientHandler) : S3FileSystemProvider() { + + + override fun getScheme(): String? { + return "s3" + } + + override fun getOutputStream(path: S3Path): OutputStream { + return createStreamer(path) + } + + override fun getInputStream(path: S3Path): InputStream { + val client = clientHandler.getClientForBucket(path.bucketName) + return client.getObject(path.bucketName, path.objectName).objectContent + } + + private fun createStreamer(path: S3Path): OutputStream { + val pis = PipedInputStream() + val pos = PipedOutputStream(pis) + val exception = AtomicReference() + + val thread = Thread.ofVirtual().start { + try { + val client = clientHandler.getClientForBucket(path.bucketName) + client.putObject(PutObjectRequest(path.bucketName, path.objectName, pis, ObjectMetadata())) + } catch (e: Exception) { + exception.set(e) + } finally { + IOUtils.closeQuietly(pis) + } + } + + return object : OutputStream() { + override fun write(b: Int) { + val exception = exception.get() + if (exception != null) throw exception + pos.write(b) + } + + override fun close() { + pos.close() + if (thread.isAlive) thread.join() + } + } + } + + override fun fetchChildren(path: S3Path): MutableList { + val paths = mutableListOf() + + // root + if (path.isRoot) { + for (bucket in clientHandler.buckets) { + val p = path.resolve(bucket.name) + p.attributes = S3FileAttributes( + directory = true, + lastModifiedTime = bucket.creationDate.toInstant().toEpochMilli() + ) + paths.add(p) + } + return paths + } + + var nextMarker = StringUtils.EMPTY + val maxKeys = 100 + val bucketName = path.bucketName + while (true) { + val request = ListObjectsRequest() + .withBucketName(bucketName) + .withMaxKeys(maxKeys) + .withDelimiter(path.fileSystem.separator) + + if (path.objectName.isNotBlank()) request.withPrefix(path.objectName + path.fileSystem.separator) + if (nextMarker.isNotBlank()) request.withMarker(nextMarker) + + + val objectListing = clientHandler.getClientForBucket(bucketName).listObjects(request) + for (e in objectListing.commonPrefixes) { + val p = path.bucket.resolve(e) + p.attributes = p.attributes.copy(directory = true) + delete(p) + paths.add(p) + } + + for (e in objectListing.objectSummaries) { + val p = path.bucket.resolve(e.key) + p.attributes = p.attributes.copy( + regularFile = true, size = e.size, + lastModifiedTime = e.lastModified.time + ) + paths.add(p) + } + + if (objectListing.isTruncated.not()) { + break + } + + nextMarker = objectListing.nextMarker + + } + + paths.addAll(directories[path.absolutePathString()] ?: emptyList()) + + return paths + } + + override fun delete(path: S3Path, isDirectory: Boolean) { + if (isDirectory.not()) + clientHandler.getClientForBucket(path.bucketName).deleteObject(path.bucketName, path.objectName) + } + + override fun checkAccess(path: S3Path, vararg modes: AccessMode) { + try { + val client = clientHandler.getClientForBucket(path.bucketName) + if (client.doesObjectExist(path.bucketName, path.objectName).not()) { + throw NoSuchFileException(path.objectName) + } + } catch (e: Exception) { + if (e is NoSuchFileException) throw e + throw NoSuchFileException(e.message) + } + } + +} \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSHostOptionsPane.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSHostOptionsPane.kt new file mode 100644 index 0000000..571fd0e --- /dev/null +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSHostOptionsPane.kt @@ -0,0 +1,313 @@ +package app.termora.plugins.cos + +import app.termora.* +import app.termora.plugin.internal.BasicProxyOption +import com.formdev.flatlaf.FlatClientProperties +import com.formdev.flatlaf.ui.FlatTextBorder +import com.jgoodies.forms.builder.FormBuilder +import com.jgoodies.forms.layout.FormLayout +import org.apache.commons.lang3.StringUtils +import java.awt.BorderLayout +import java.awt.KeyboardFocusManager +import java.awt.event.ComponentAdapter +import java.awt.event.ComponentEvent +import javax.swing.* + +class COSHostOptionsPane : OptionsPane() { + private val generalOption = GeneralOption() + private val proxyOption = BasicProxyOption(listOf(ProxyType.HTTP)) + private val sftpOption = SFTPOption() + + init { + addOption(generalOption) + addOption(proxyOption) + addOption(sftpOption) + + } + + fun getHost(): Host { + val name = generalOption.nameTextField.text + val protocol = COSProtocolProvider.PROTOCOL + val port = 0 + var authentication = Authentication.Companion.No + var proxy = Proxy.Companion.No + val authenticationType = AuthenticationType.Password + + authentication = authentication.copy( + type = authenticationType, + password = String(generalOption.passwordTextField.password) + ) + + + if (proxyOption.proxyTypeComboBox.selectedItem != ProxyType.No) { + proxy = proxy.copy( + type = proxyOption.proxyTypeComboBox.selectedItem as ProxyType, + host = proxyOption.proxyHostTextField.text, + username = proxyOption.proxyUsernameTextField.text, + password = String(proxyOption.proxyPasswordTextField.password), + port = proxyOption.proxyPortTextField.value as Int, + authenticationType = proxyOption.proxyAuthenticationTypeComboBox.selectedItem as AuthenticationType, + ) + } + + + val options = Options.Default.copy( + sftpDefaultDirectory = sftpOption.defaultDirectoryField.text, + extras = mutableMapOf( + "cos.delimiter" to generalOption.delimiterTextField.text, + ) + ) + + return Host( + name = name, + protocol = protocol, + port = port, + username = generalOption.usernameTextField.text, + authentication = authentication, + proxy = proxy, + sort = System.currentTimeMillis(), + remark = generalOption.remarkTextArea.text, + options = options, + ) + } + + fun setHost(host: Host) { + generalOption.nameTextField.text = host.name + generalOption.usernameTextField.text = host.username + generalOption.remarkTextArea.text = host.remark + generalOption.passwordTextField.text = host.authentication.password + generalOption.delimiterTextField.text = host.options.extras["cos.delimiter"] ?: StringUtils.EMPTY + + proxyOption.proxyTypeComboBox.selectedItem = host.proxy.type + proxyOption.proxyHostTextField.text = host.proxy.host + proxyOption.proxyPasswordTextField.text = host.proxy.password + proxyOption.proxyUsernameTextField.text = host.proxy.username + proxyOption.proxyPortTextField.value = host.proxy.port + proxyOption.proxyAuthenticationTypeComboBox.selectedItem = host.proxy.authenticationType + + + sftpOption.defaultDirectoryField.text = host.options.sftpDefaultDirectory + } + + fun validateFields(): Boolean { + val host = getHost() + + // general + if (validateField(generalOption.nameTextField)) { + return false + } + + if (validateField(generalOption.usernameTextField)) { + return false + } + + if (host.authentication.type == AuthenticationType.Password) { + if (validateField(generalOption.passwordTextField)) { + return false + } + } + + // proxy + if (host.proxy.type != ProxyType.No) { + if (validateField(proxyOption.proxyHostTextField) + ) { + return false + } + + if (host.proxy.authenticationType != AuthenticationType.No) { + if (validateField(proxyOption.proxyUsernameTextField) + || validateField(proxyOption.proxyPasswordTextField) + ) { + return false + } + } + } + + return true + } + + /** + * 返回 true 表示有错误 + */ + private fun validateField(textField: JTextField): Boolean { + if (textField.isEnabled && textField.text.isBlank()) { + setOutlineError(textField) + return true + } + return false + } + + private fun setOutlineError(c: JComponent) { + selectOptionJComponent(c) + c.putClientProperty(FlatClientProperties.OUTLINE, FlatClientProperties.OUTLINE_ERROR) + c.requestFocusInWindow() + } + + + private inner class GeneralOption : JPanel(BorderLayout()), Option { + val nameTextField = OutlineTextField(128) + val usernameTextField = OutlineTextField(128) + val passwordTextField = OutlinePasswordField(255) + val remarkTextArea = FixedLengthTextArea(512) + + // val regionComboBox = OutlineComboBox() + val delimiterTextField = OutlineTextField(128) + + init { + initView() + initEvents() + } + + private fun initView() { + + /*regionComboBox.addItem("ap-beijing-1") + regionComboBox.addItem("ap-beijing") + regionComboBox.addItem("ap-nanjing") + regionComboBox.addItem("ap-shanghai") + regionComboBox.addItem("ap-guangzhou") + regionComboBox.addItem("ap-chengdu") + regionComboBox.addItem("ap-chongqing") + regionComboBox.addItem("ap-shenzhen-fsi") + regionComboBox.addItem("ap-shanghai-fsi") + regionComboBox.addItem("ap-beijing-fsi") + + regionComboBox.addItem("ap-hongkong") + regionComboBox.addItem("ap-singapore") + regionComboBox.addItem("ap-jakarta") + regionComboBox.addItem("ap-seoul") + regionComboBox.addItem("ap-bangkok") + regionComboBox.addItem("ap-tokyo") + regionComboBox.addItem("na-siliconvalley") + regionComboBox.addItem("na-ashburn") + regionComboBox.addItem("sa-saopaulo") + regionComboBox.addItem("eu-frankfurt") + + regionComboBox.isEditable = true*/ + + delimiterTextField.text = "/" + delimiterTextField.isEditable = false + add(getCenterComponent(), BorderLayout.CENTER) + } + + private fun initEvents() { + + addComponentListener(object : ComponentAdapter() { + override fun componentResized(e: ComponentEvent) { + SwingUtilities.invokeLater { nameTextField.requestFocusInWindow() } + removeComponentListener(this) + } + }) + } + + + override fun getIcon(isSelected: Boolean): Icon { + return Icons.settings + } + + override fun getTitle(): String { + return I18n.getString("termora.new-host.general") + } + + override fun getJComponent(): JComponent { + return this + } + + private fun getCenterComponent(): JComponent { + val layout = FormLayout( + "left:pref, $FORM_MARGIN, default:grow, $FORM_MARGIN, pref, $FORM_MARGIN, default", + "pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref" + ) + remarkTextArea.setFocusTraversalKeys( + KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS, + KeyboardFocusManager.getCurrentKeyboardFocusManager() + .getDefaultFocusTraversalKeys(KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS) + ) + remarkTextArea.setFocusTraversalKeys( + KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS, + KeyboardFocusManager.getCurrentKeyboardFocusManager() + .getDefaultFocusTraversalKeys(KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS) + ) + + remarkTextArea.rows = 8 + remarkTextArea.lineWrap = true + remarkTextArea.border = BorderFactory.createEmptyBorder(4, 4, 4, 4) + + + var rows = 1 + val step = 2 + val panel = FormBuilder.create().layout(layout) + .add("${I18n.getString("termora.new-host.general.name")}:").xy(1, rows) + .add(nameTextField).xyw(3, rows, 5).apply { rows += step } + + .add("SecretId:").xy(1, rows) + .add(usernameTextField).xyw(3, rows, 5).apply { rows += step } + + .add("SecretKey:").xy(1, rows) + .add(passwordTextField).xyw(3, rows, 5).apply { rows += step } + + .add("Delimiter:").xy(1, rows) + .add(delimiterTextField).xyw(3, rows, 5).apply { rows += step } + + .add("${I18n.getString("termora.new-host.general.remark")}:").xy(1, rows) + .add(JScrollPane(remarkTextArea).apply { border = FlatTextBorder() }) + .xyw(3, rows, 5).apply { rows += step } + + .build() + + + return panel + } + + } + + + private inner class SFTPOption : JPanel(BorderLayout()), Option { + val defaultDirectoryField = OutlineTextField(255) + + + init { + initView() + initEvents() + } + + private fun initView() { + add(getCenterComponent(), BorderLayout.CENTER) + } + + private fun initEvents() { + + } + + + override fun getIcon(isSelected: Boolean): Icon { + return Icons.folder + } + + override fun getTitle(): String { + return I18n.getString("termora.transport.sftp") + } + + override fun getJComponent(): JComponent { + return this + } + + private fun getCenterComponent(): JComponent { + val layout = FormLayout( + "left:pref, $FORM_MARGIN, default:grow", + "pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref" + ) + + var rows = 1 + val step = 2 + val panel = FormBuilder.create().layout(layout) + .add("${I18n.getString("termora.settings.sftp.default-directory")}:").xy(1, rows) + .add(defaultDirectoryField).xy(3, rows).apply { rows += step } + .build() + + + return panel + } + } + + +} \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSPlugin.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSPlugin.kt index 0e71228..2df00ee 100644 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSPlugin.kt +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSPlugin.kt @@ -1,8 +1,5 @@ package app.termora.plugins.cos -import app.termora.DynamicIcon -import app.termora.I18n -import app.termora.Icons import app.termora.plugin.Extension import app.termora.plugin.ExtensionSupport import app.termora.plugin.PaidPlugin @@ -13,8 +10,8 @@ class COSPlugin : PaidPlugin { private val support = ExtensionSupport() init { - support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.Companion.instance } - support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.Companion.instance } + support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.instance } + support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.instance } } override fun getAuthor(): String { diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanel.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanel.kt index c7e11fa..0ee86e0 100644 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanel.kt +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanel.kt @@ -1,22 +1,36 @@ package app.termora.plugins.cos +import app.termora.Disposer import app.termora.Host import app.termora.protocol.ProtocolHostPanel -import org.apache.commons.lang3.StringUtils +import java.awt.BorderLayout class COSProtocolHostPanel : ProtocolHostPanel() { + + private val pane = COSHostOptionsPane() + + init { + initView() + initEvents() + } + + + private fun initView() { + add(pane, BorderLayout.CENTER) + Disposer.register(this, pane) + } + + private fun initEvents() {} + override fun getHost(): Host { - return Host( - name = StringUtils.EMPTY, - protocol = COSProtocolProvider.Companion.PROTOCOL - ) + return pane.getHost() } override fun setHost(host: Host) { - + pane.setHost(host) } override fun validateFields(): Boolean { - return true + return pane.validateFields() } } \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanelExtension.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanelExtension.kt index 61aa7f0..5c29bed 100644 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanelExtension.kt +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolHostPanelExtension.kt @@ -10,7 +10,7 @@ class COSProtocolHostPanelExtension private constructor() : ProtocolHostPanelExt } override fun getProtocolProvider(): ProtocolProvider { - return COSProtocolProvider.Companion.instance + return COSProtocolProvider.instance } override fun createProtocolHostPanel(): ProtocolHostPanel { diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProvider.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProvider.kt index 410deca..5c2d8cf 100644 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProvider.kt +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProvider.kt @@ -2,10 +2,14 @@ package app.termora.plugins.cos import app.termora.DynamicIcon import app.termora.Icons -import app.termora.protocol.FileObjectHandler -import app.termora.protocol.FileObjectRequest +import app.termora.protocol.PathHandler +import app.termora.protocol.PathHandlerRequest import app.termora.protocol.TransferProtocolProvider -import org.apache.commons.vfs2.provider.FileProvider +import com.qcloud.cos.ClientConfig +import com.qcloud.cos.auth.BasicCOSCredentials +import com.qcloud.cos.model.Bucket +import org.apache.commons.lang3.StringUtils + class COSProtocolProvider private constructor() : TransferProtocolProvider { @@ -22,12 +26,30 @@ class COSProtocolProvider private constructor() : TransferProtocolProvider { return Icons.tencent } - override fun getFileProvider(): FileProvider { - return COSFileProvider.instance + override fun createPathHandler(requester: PathHandlerRequest): PathHandler { + val host = requester.host + val secretId = host.username + val secretKey = host.authentication.password + val cred = BasicCOSCredentials(secretId, secretKey) + val clientConfig = ClientConfig() + + clientConfig.isPrintShutdownStackTrace = false + val cosClient = COSClientHandler.createCOSClient(cred, StringUtils.EMPTY, host.proxy) + val buckets: List + + try { + buckets = cosClient.listBuckets() + if (buckets.isEmpty()) { + throw IllegalStateException("没有获取到桶信息") + } + } finally { + cosClient.shutdown() + } + + val defaultPath = host.options.sftpDefaultDirectory + val fs = COSFileSystem(COSClientHandler(cred, host.proxy, buckets)) + return PathHandler(fs, fs.getPath(defaultPath)) } - override fun getRootFileObject(requester: FileObjectRequest): FileObjectHandler { - TODO("Not yet implemented") - } } \ No newline at end of file diff --git a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProviderExtension.kt b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProviderExtension.kt index ee3bc35..942f070 100644 --- a/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProviderExtension.kt +++ b/plugins/cos/src/main/kotlin/app/termora/plugins/cos/COSProtocolProviderExtension.kt @@ -9,6 +9,6 @@ class COSProtocolProviderExtension private constructor() : ProtocolProviderExten } override fun getProtocolProvider(): ProtocolProvider { - return COSProtocolProvider.Companion.instance + return COSProtocolProvider.instance } } \ No newline at end of file diff --git a/plugins/s3/build.gradle.kts b/plugins/s3/build.gradle.kts index c963d55..43801a6 100644 --- a/plugins/s3/build.gradle.kts +++ b/plugins/s3/build.gradle.kts @@ -3,7 +3,7 @@ plugins { } -project.version = "0.0.4" +project.version = "0.0.5" dependencies { diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystem.kt b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystem.kt new file mode 100644 index 0000000..9d583d1 --- /dev/null +++ b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystem.kt @@ -0,0 +1,13 @@ +package app.termora.plugins.s3 + +import app.termora.transfer.s3.S3FileSystem +import io.minio.MinioClient + +class MyS3FileSystem(private val minioClient: MinioClient) : + S3FileSystem(MyS3FileSystemProvider(minioClient)) { + + override fun close() { + minioClient.close() + super.close() + } +} diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystemProvider.kt b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystemProvider.kt new file mode 100644 index 0000000..95b43ed --- /dev/null +++ b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/MyS3FileSystemProvider.kt @@ -0,0 +1,157 @@ +package app.termora.plugins.s3 + +import app.termora.transfer.s3.S3FileAttributes +import app.termora.transfer.s3.S3FileSystemProvider +import app.termora.transfer.s3.S3Path +import io.minio.* +import io.minio.errors.ErrorResponseException +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.StringUtils +import java.io.InputStream +import java.io.OutputStream +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.nio.file.AccessMode +import java.nio.file.NoSuchFileException +import java.util.concurrent.atomic.AtomicReference +import kotlin.io.path.absolutePathString + +class MyS3FileSystemProvider(private val minioClient: MinioClient) : S3FileSystemProvider() { + + override fun getScheme(): String? { + return "s3" + } + + override fun getOutputStream(path: S3Path): OutputStream { + return createStreamer(path) + } + + override fun getInputStream(path: S3Path): InputStream { + return minioClient.getObject( + GetObjectArgs.builder().bucket(path.bucketName) + .`object`(path.objectName).build() + ) + } + + + private fun createStreamer(path: S3Path): OutputStream { + val pis = PipedInputStream() + val pos = PipedOutputStream(pis) + val exception = AtomicReference() + + val thread = Thread.ofVirtual().start { + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(path.bucketName) + .stream(pis, -1, 32 * 1024 * 1024) + .`object`(path.objectName).build() + ) + } catch (e: Exception) { + exception.set(e) + } finally { + IOUtils.closeQuietly(pis) + } + } + + return object : OutputStream() { + override fun write(b: Int) { + val exception = exception.get() + if (exception != null) throw exception + pos.write(b) + } + + override fun close() { + pos.close() + if (thread.isAlive) thread.join() + } + } + } + + override fun fetchChildren(path: S3Path): MutableList { + val paths = mutableListOf() + + // root + if (path.isRoot) { + for (bucket in minioClient.listBuckets()) { + val p = path.resolve(bucket.name()) + p.attributes = S3FileAttributes( + directory = true, + lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli() + ) + paths.add(p) + } + return paths + } + + var startAfter = StringUtils.EMPTY + val maxKeys = 100 + + while (true) { + val builder = ListObjectsArgs.builder() + .bucket(path.bucketName) + .maxKeys(maxKeys) + .delimiter(path.fileSystem.separator) + + if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator) + if (startAfter.isNotBlank()) builder.startAfter(startAfter) + + + val subPaths = mutableListOf() + for (e in minioClient.listObjects(builder.build())) { + val item = e.get() + val p = path.bucket.resolve(item.objectName()) + var attributes = p.attributes.copy( + directory = item.isDir, + regularFile = item.isDir.not(), + size = item.size() + ) + if (item.lastModified() != null) { + attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli()) + } + p.attributes = attributes + + // 如果是文件夹，那么就要删除内存中的 + if (attributes.isDirectory) { + delete(p) + } + + subPaths.add(p) + startAfter = item.objectName() + } + + paths.addAll(subPaths) + + if (subPaths.size < maxKeys) + break + + + } + + paths.addAll(directories[path.absolutePathString()] ?: emptyList()) + + return paths + } + + + override fun delete(path: S3Path, isDirectory: Boolean) { + if (isDirectory.not()) + minioClient.removeObject( + RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build() + ) + } + + + override fun checkAccess(path: S3Path, vararg modes: AccessMode) { + try { + minioClient.statObject( + StatObjectArgs.builder() + .`object`(path.objectName) + .bucket(path.bucketName).build() + ) + } catch (e: ErrorResponseException) { + throw NoSuchFileException(e.errorResponse().message()) + } + } + +} \ No newline at end of file diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ProtocolProvider.kt b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ProtocolProvider.kt index 192e7b6..c631b59 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ProtocolProvider.kt +++ b/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ProtocolProvider.kt @@ -70,7 +70,7 @@ class S3ProtocolProvider private constructor() : TransferProtocolProvider { // val delimiter = host.options.extras["s3.delimiter"] ?: "/" val defaultPath = host.options.sftpDefaultDirectory val minioClient = builder.build() - val fs = S3FileSystem(minioClient) + val fs = MyS3FileSystem(minioClient) return PathHandler(fs, fs.getPath(defaultPath)) } diff --git a/plugins/s3/src/test/kotlin/app/termora/plugins/s3/S3FileSystemTest.kt b/plugins/s3/src/test/kotlin/app/termora/plugins/s3/S3FileSystemTest.kt index 23f1328..f51314e 100644 --- a/plugins/s3/src/test/kotlin/app/termora/plugins/s3/S3FileSystemTest.kt +++ b/plugins/s3/src/test/kotlin/app/termora/plugins/s3/S3FileSystemTest.kt @@ -58,7 +58,7 @@ class S3FileSystemTest { } } - val fileSystem = S3FileSystem(minioClient) + val fileSystem = MyS3FileSystem(minioClient) val path = fileSystem.getPath("/") PathWalker.walkFileTree(path, object : PathVisitor { override fun preVisitDirectory( diff --git a/settings.gradle.kts b/settings.gradle.kts index 65ea550..1824f2b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ rootProject.name = "termora" include("plugins:s3") //include("plugins:oss") -//include("plugins:cos") +include("plugins:cos") //include("plugins:obs") //include("plugins:ftp") include("plugins:bg") diff --git a/src/main/kotlin/app/termora/plugin/internal/BasicProxyOption.kt b/src/main/kotlin/app/termora/plugin/internal/BasicProxyOption.kt index efddf7e..437e39b 100644 --- a/src/main/kotlin/app/termora/plugin/internal/BasicProxyOption.kt +++ b/src/main/kotlin/app/termora/plugin/internal/BasicProxyOption.kt @@ -10,7 +10,8 @@ import java.awt.Component import java.awt.event.ItemEvent import javax.swing.* -class BasicProxyOption : JPanel(BorderLayout()), Option { +class BasicProxyOption(private val proxyTypes: List = listOf(ProxyType.HTTP, ProxyType.SOCKS5)) : + JPanel(BorderLayout()), Option { private val formMargin = "7dlu" val proxyTypeComboBox = FlatComboBox() @@ -61,8 +62,9 @@ class BasicProxyOption : JPanel(BorderLayout()), Option { } proxyTypeComboBox.addItem(ProxyType.No) - proxyTypeComboBox.addItem(ProxyType.HTTP) - proxyTypeComboBox.addItem(ProxyType.SOCKS5) + for (type in proxyTypes) { + proxyTypeComboBox.addItem(type) + } proxyAuthenticationTypeComboBox.addItem(AuthenticationType.No) proxyAuthenticationTypeComboBox.addItem(AuthenticationType.Password) diff --git a/src/main/kotlin/app/termora/protocol/PathHandler.kt b/src/main/kotlin/app/termora/protocol/PathHandler.kt index 5262956..4312bf6 100644 --- a/src/main/kotlin/app/termora/protocol/PathHandler.kt +++ b/src/main/kotlin/app/termora/protocol/PathHandler.kt @@ -1,9 +1,12 @@ package app.termora.protocol import app.termora.Disposable +import org.apache.commons.io.IOUtils import java.nio.file.FileSystem import java.nio.file.Path open class PathHandler(val fileSystem: FileSystem, val path: Path) : Disposable { - + override fun dispose() { + IOUtils.closeQuietly(fileSystem) + } } \ No newline at end of file diff --git a/src/main/kotlin/app/termora/terminal/panel/vw/TransferVisualWindow.kt b/src/main/kotlin/app/termora/terminal/panel/vw/TransferVisualWindow.kt index 0969800..11ad4cc 100644 --- a/src/main/kotlin/app/termora/terminal/panel/vw/TransferVisualWindow.kt +++ b/src/main/kotlin/app/termora/terminal/panel/vw/TransferVisualWindow.kt @@ -442,11 +442,6 @@ class TransferVisualWindow(tab: SSHTerminalTab, visualWindowManager: VisualWindo override fun getWorkdir(): Path? { return transportPanel.workdir } - - override fun getTableModel(): TransportTableModel? { - return transportPanel.getTableModel() - } - } } @@ -457,7 +452,6 @@ class TransferVisualWindow(tab: SSHTerminalTab, visualWindowManager: VisualWindo transferManager, object : DefaultInternalTransferManager.WorkdirProvider { override fun getWorkdir() = null - override fun getTableModel() = null }, createWorkdirProvider(transportPanel) ) diff --git a/src/main/kotlin/app/termora/transfer/DefaultInternalTransferManager.kt b/src/main/kotlin/app/termora/transfer/DefaultInternalTransferManager.kt index 1bae3aa..da9aaa2 100644 --- a/src/main/kotlin/app/termora/transfer/DefaultInternalTransferManager.kt +++ b/src/main/kotlin/app/termora/transfer/DefaultInternalTransferManager.kt @@ -31,6 +31,7 @@ import kotlin.collections.ArrayDeque import kotlin.collections.List import kotlin.collections.Set import kotlin.collections.isNotEmpty +import kotlin.io.path.exists import kotlin.io.path.name import kotlin.io.path.pathString import kotlin.math.max @@ -49,7 +50,6 @@ class DefaultInternalTransferManager( interface WorkdirProvider { fun getWorkdir(): Path? - fun getTableModel(): TransportTableModel? } @@ -85,19 +85,14 @@ class DefaultInternalTransferManager( if (paths.isEmpty()) return CompletableFuture.completedFuture(Unit) val future = CompletableFuture() - val tableModel = when (targetWorkdir.fileSystem) { - source.getWorkdir()?.fileSystem -> source.getTableModel() - target.getWorkdir()?.fileSystem -> target.getTableModel() - else -> null - } coroutineScope.launch(Dispatchers.IO) { try { val context = AskTransferContext(TransferAction.Overwrite, false) for (pair in paths) { - if (mode == TransferMode.Transfer && tableModel != null && context.applyAll.not()) { + if (mode == TransferMode.Transfer && context.applyAll.not()) { val action = withContext(Dispatchers.Swing) { - getTransferAction(context, tableModel, pair.second) + getTransferAction(context, targetWorkdir.resolve(pair.first.name), pair.second) } if (action == null) { break @@ -143,21 +138,19 @@ class DefaultInternalTransferManager( private fun getTransferAction( context: AskTransferContext, - model: TransportTableModel, + path: Path, source: TransportTableModel.Attributes ): TransferAction? { if (context.applyAll) return context.action - for (i in 0 until model.rowCount) { - val c = model.getAttributes(i) - if (c.name != source.name) continue - val transfer = askTransfer(source, c) + if (path.exists()) { + val transfer = askTransfer(source, source) context.action = transfer.action context.applyAll = transfer.applyAll if (transfer.option != JOptionPane.OK_OPTION) return null } - return context.action + return TransferAction.Overwrite } diff --git a/src/main/kotlin/app/termora/transfer/FileTransfer.kt b/src/main/kotlin/app/termora/transfer/FileTransfer.kt index 8ab93ee..be96843 100644 --- a/src/main/kotlin/app/termora/transfer/FileTransfer.kt +++ b/src/main/kotlin/app/termora/transfer/FileTransfer.kt @@ -6,6 +6,7 @@ import java.io.InputStream import java.io.OutputStream import java.nio.file.Path import java.nio.file.StandardOpenOption +import java.util.concurrent.atomic.AtomicBoolean import kotlin.io.path.inputStream import kotlin.io.path.outputStream @@ -18,8 +19,12 @@ class FileTransfer( private lateinit var input: InputStream private lateinit var output: OutputStream + private val closed = AtomicBoolean(false) + override suspend fun transfer(bufferSize: Int): Long { + if (closed.get()) throw IllegalStateException("Transfer already closed") + if (::input.isInitialized.not()) { input = source().inputStream(StandardOpenOption.READ) } @@ -48,12 +53,14 @@ class FileTransfer( } override fun close() { - if (::input.isInitialized) { - IOUtils.closeQuietly(input) - } + if (closed.compareAndSet(false, true)) { + if (::input.isInitialized) { + IOUtils.closeQuietly(input) + } - if (::output.isInitialized) { - IOUtils.closeQuietly(output) + if (::output.isInitialized) { + IOUtils.closeQuietly(output) + } } } diff --git a/src/main/kotlin/app/termora/transfer/TransferTableModel.kt b/src/main/kotlin/app/termora/transfer/TransferTableModel.kt index 05a035e..1890fad 100644 --- a/src/main/kotlin/app/termora/transfer/TransferTableModel.kt +++ b/src/main/kotlin/app/termora/transfer/TransferTableModel.kt @@ -424,6 +424,11 @@ class TransferTableModel(private val coroutineScope: CoroutineScope) : // 异步上报，因为数据量非常大，所以采用异步 reporter.report(node, len, System.currentTimeMillis()) } + + // 因为可能是异步传输，只有关闭后才能确保数据已经到达云端 + // 尤其是 S3 协议 + if (transfer is Closeable) IOUtils.closeQuietly(transfer) + withContext(Dispatchers.Swing) { if (continueTransfer(node)) { changeState(node, State.Done) diff --git a/src/main/kotlin/app/termora/transfer/TransportPanel.kt b/src/main/kotlin/app/termora/transfer/TransportPanel.kt index 1d1fab9..35316da 100644 --- a/src/main/kotlin/app/termora/transfer/TransportPanel.kt +++ b/src/main/kotlin/app/termora/transfer/TransportPanel.kt @@ -290,7 +290,7 @@ class TransportPanel( // 传输完成之后刷新 transferManager.addTransferListener(object : TransferListener { override fun onTransferChanged(transfer: Transfer, state: TransferTreeTableNode.State) { - if (state != TransferTreeTableNode.State.Done) return + if (state != TransferTreeTableNode.State.Done && state != TransferTreeTableNode.State.Failed) return if (transfer.target().fileSystem != _fileSystem) return if (transfer.target() == workdir || transfer.target().parent == workdir) { reload(requestFocus = false) diff --git a/src/main/kotlin/app/termora/transfer/TransportViewer.kt b/src/main/kotlin/app/termora/transfer/TransportViewer.kt index 44cdadf..953d3f2 100644 --- a/src/main/kotlin/app/termora/transfer/TransportViewer.kt +++ b/src/main/kotlin/app/termora/transfer/TransportViewer.kt @@ -95,9 +95,6 @@ class TransportViewer : JPanel(BorderLayout()), DataProvider, Disposable { return source.getSelectedTransportPanel()?.workdir } - override fun getTableModel(): TransportTableModel? { - return source.getSelectedTransportPanel()?.getTableModel() - } }, object : DefaultInternalTransferManager.WorkdirProvider { @@ -105,9 +102,6 @@ class TransportViewer : JPanel(BorderLayout()), DataProvider, Disposable { return target.getSelectedTransportPanel()?.workdir } - override fun getTableModel(): TransportTableModel? { - return target.getSelectedTransportPanel()?.getTableModel() - } }) } diff --git a/src/main/kotlin/app/termora/transfer/internal/sftp/SFTPPathHandler.kt b/src/main/kotlin/app/termora/transfer/internal/sftp/SFTPPathHandler.kt index 88c2cc3..e14ecf0 100644 --- a/src/main/kotlin/app/termora/transfer/internal/sftp/SFTPPathHandler.kt +++ b/src/main/kotlin/app/termora/transfer/internal/sftp/SFTPPathHandler.kt @@ -24,8 +24,8 @@ internal class SFTPPathHandler( } override fun dispose() { + super.dispose() session.removeCloseFutureListener(listener) - IOUtils.closeQuietly(fileSystem) IOUtils.closeQuietly(session) IOUtils.closeQuietly(client) } diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileAttributes.kt b/src/main/kotlin/app/termora/transfer/s3/S3FileAttributes.kt similarity index 97% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileAttributes.kt rename to src/main/kotlin/app/termora/transfer/s3/S3FileAttributes.kt index b70687a..74b1c1d 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileAttributes.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3FileAttributes.kt @@ -1,4 +1,4 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileTime diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystem.kt b/src/main/kotlin/app/termora/transfer/s3/S3FileSystem.kt similarity index 77% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystem.kt rename to src/main/kotlin/app/termora/transfer/s3/S3FileSystem.kt index fdf6117..514770d 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystem.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3FileSystem.kt @@ -1,14 +1,11 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 -import io.minio.MinioClient import org.apache.sshd.common.file.util.BaseFileSystem import java.nio.file.Path import java.nio.file.attribute.UserPrincipalLookupService import java.util.concurrent.atomic.AtomicBoolean -class S3FileSystem( - private val minioClient: MinioClient, -) : BaseFileSystem(S3FileSystemProvider(minioClient)) { +open class S3FileSystem(provider: S3FileSystemProvider) : BaseFileSystem(provider) { private val isOpen = AtomicBoolean(true) @@ -20,16 +17,14 @@ class S3FileSystem( return path } - override fun close() { - if (isOpen.compareAndSet(false, true)) { - minioClient.close() - } - } - override fun isOpen(): Boolean { return isOpen.get() } + override fun close() { + isOpen.compareAndSet(false, true) + } + override fun getRootDirectories(): Iterable { return mutableSetOf(create(separator)) } diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystemProvider.kt b/src/main/kotlin/app/termora/transfer/s3/S3FileSystemProvider.kt similarity index 50% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystemProvider.kt rename to src/main/kotlin/app/termora/transfer/s3/S3FileSystemProvider.kt index 6ba483a..f123642 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3FileSystemProvider.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3FileSystemProvider.kt @@ -1,32 +1,21 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 -import io.minio.* -import io.minio.errors.ErrorResponseException -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils +import java.io.InputStream import java.io.OutputStream -import java.io.PipedInputStream -import java.io.PipedOutputStream import java.net.URI -import java.nio.channels.Channels import java.nio.channels.SeekableByteChannel import java.nio.file.* import java.nio.file.attribute.* import java.nio.file.spi.FileSystemProvider -import java.util.concurrent.atomic.AtomicReference import kotlin.io.path.absolutePathString import kotlin.io.path.name -class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemProvider() { +abstract class S3FileSystemProvider() : FileSystemProvider() { /** * 因为 S3 协议不存在文件夹，所以用户新建的文件夹先保存到内存中 */ - private val directories = mutableMapOf>() - - override fun getScheme(): String? { - return "s3" - } + protected val directories = mutableMapOf>() override fun newFileSystem( uri: URI, @@ -49,51 +38,15 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro vararg attrs: FileAttribute<*> ): SeekableByteChannel { if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path") - if (options.contains(StandardOpenOption.WRITE)) { - return S3WriteSeekableByteChannel(Channels.newChannel(createStreamer(path))) + return if (options.contains(StandardOpenOption.WRITE)) { + S3WriteSeekableByteChannel(getOutputStream(path)) } else { - val response = minioClient.getObject( - GetObjectArgs.builder().bucket(path.bucketName) - .`object`(path.objectName).build() - ) - return S3ReadSeekableByteChannel(Channels.newChannel(response), stat(path)) + S3ReadSeekableByteChannel(getInputStream(path), path.attributes.size()) } } - - private fun createStreamer(path: S3Path): OutputStream { - val pis = PipedInputStream() - val pos = PipedOutputStream(pis) - val exception = AtomicReference() - - val thread = Thread.ofVirtual().start { - try { - minioClient.putObject( - PutObjectArgs.builder() - .bucket(path.bucketName) - .stream(pis, -1, 32 * 1024 * 1024) - .`object`(path.objectName).build() - ) - } catch (e: Exception) { - exception.set(e) - } finally { - IOUtils.closeQuietly(pis) - } - } - - return object : OutputStream() { - override fun write(b: Int) { - val exception = exception.get() - if (exception != null) throw exception - pos.write(b) - } - - override fun close() { - pos.close() - if (thread.isAlive) thread.join() - } - } - } + abstract fun getOutputStream(path: S3Path): OutputStream + abstract fun getInputStream(path: S3Path): InputStream override fun newDirectoryStream( dir: Path, @@ -101,7 +54,7 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro ): DirectoryStream { return object : DirectoryStream { override fun iterator(): MutableIterator { - return files(dir as S3Path).iterator() + return fetchChildren(dir as S3Path).iterator() } override fun close() { @@ -110,70 +63,8 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro } } - private fun files(path: S3Path): MutableList { - val paths = mutableListOf() + abstract fun fetchChildren(path: S3Path): MutableList - // root - if (path.isRoot) { - for (bucket in minioClient.listBuckets()) { - val p = path.resolve(bucket.name()) - p.attributes = S3FileAttributes( - directory = true, - lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli() - ) - paths.add(p) - } - return paths - } - - var startAfter = StringUtils.EMPTY - val maxKeys = 100 - - while (true) { - val builder = ListObjectsArgs.builder() - .bucket(path.bucketName) - .maxKeys(maxKeys) - .delimiter(path.fileSystem.separator) - - if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator) - if (startAfter.isNotBlank()) builder.startAfter(startAfter) - - - val subPaths = mutableListOf() - for (e in minioClient.listObjects(builder.build())) { - val item = e.get() - val p = path.bucket.resolve(item.objectName()) - var attributes = p.attributes.copy( - directory = item.isDir, - regularFile = item.isDir.not(), - size = item.size() - ) - if (item.lastModified() != null) { - attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli()) - } - p.attributes = attributes - - // 如果是文件夹，那么就要删除内存中的 - if (attributes.isDirectory) { - delete(p) - } - - subPaths.add(p) - startAfter = item.objectName() - } - - paths.addAll(subPaths) - - if (subPaths.size < maxKeys) - break - - - } - - paths.addAll(directories[path.absolutePathString()] ?: emptyList()) - - return paths - } override fun createDirectory(dir: Path, vararg attrs: FileAttribute<*>) { synchronized(this) { @@ -196,13 +87,20 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro directories[parent.absolutePathString()]?.removeIf { it.name == path.name } } } - return + delete(path, true) + } else { + delete(path, false) } - minioClient.removeObject( - RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build() - ) } + override fun checkAccess(path: Path, vararg modes: AccessMode) { + if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path") + checkAccess(path, *modes) + } + + abstract fun delete(path: S3Path, isDirectory: Boolean) + abstract fun checkAccess(path: S3Path, vararg modes: AccessMode) + override fun copy( source: Path?, target: Path?, @@ -231,25 +129,6 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro throw UnsupportedOperationException() } - override fun checkAccess(path: Path, vararg modes: AccessMode) { - if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path") - - try { - stat(path) - } catch (e: ErrorResponseException) { - throw NoSuchFileException(e.errorResponse().message()) - } - } - - - private fun stat(path: S3Path): StatObjectResponse { - return minioClient.statObject( - StatObjectArgs.builder() - .`object`(path.objectName) - .bucket(path.bucketName).build() - ) - } - override fun getFileAttributeView( path: Path, type: Class, diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3Path.kt b/src/main/kotlin/app/termora/transfer/s3/S3Path.kt similarity index 72% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3Path.kt rename to src/main/kotlin/app/termora/transfer/s3/S3Path.kt index a639b89..492ebdc 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3Path.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3Path.kt @@ -1,4 +1,4 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 import app.termora.transfer.WithPathAttributes import org.apache.sshd.common.file.util.BasePath @@ -6,7 +6,7 @@ import java.nio.file.LinkOption import java.nio.file.Path import kotlin.io.path.absolutePathString -class S3Path( +open class S3Path( fileSystem: S3FileSystem, root: String?, names: List, @@ -20,27 +20,27 @@ class S3Path( /** * 是否是 Bucket */ - val isBucket get() = parent != null && parent?.parent == null + open val isBucket get() = parent != null && parent?.parent == null /** * 是否是根 */ - val isRoot get() = absolutePathString() == separator + open val isRoot get() = absolutePathString() == separator /** * Bucket Name */ - val bucketName: String get() = names.first() + open val bucketName: String get() = names.first() /** * 获取 Bucket */ - val bucket: S3Path get() = fileSystem.getPath(root, bucketName) + open val bucket: S3Path get() = fileSystem.getPath(root, bucketName) /** * 获取所在 Bucket 的路径 */ - val objectName: String get() = names.subList(1, names.size).joinToString(separator) + open val objectName: String get() = names.subList(1, names.size).joinToString(separator) override fun getCustomType(): String? { if (isBucket) return "Bucket" diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ReadSeekableByteChannel.kt b/src/main/kotlin/app/termora/transfer/s3/S3ReadSeekableByteChannel.kt similarity index 77% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ReadSeekableByteChannel.kt rename to src/main/kotlin/app/termora/transfer/s3/S3ReadSeekableByteChannel.kt index fab6e34..dd380d3 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3ReadSeekableByteChannel.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3ReadSeekableByteChannel.kt @@ -1,16 +1,13 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 -import io.minio.StatObjectResponse import org.apache.commons.io.IOUtils +import java.io.InputStream import java.nio.ByteBuffer -import java.nio.channels.ReadableByteChannel +import java.nio.channels.Channels import java.nio.channels.SeekableByteChannel -class S3ReadSeekableByteChannel( - private val channel: ReadableByteChannel, - private val stat: StatObjectResponse -) : SeekableByteChannel { - +open class S3ReadSeekableByteChannel(input: InputStream, private val size: Long) : SeekableByteChannel { + private val channel = Channels.newChannel(input) private var position: Long = 0 override fun read(dst: ByteBuffer): Int { @@ -34,7 +31,7 @@ class S3ReadSeekableByteChannel( } override fun size(): Long { - return stat.size() + return size } override fun truncate(size: Long): SeekableByteChannel { diff --git a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3WriteSeekableByteChannel.kt b/src/main/kotlin/app/termora/transfer/s3/S3WriteSeekableByteChannel.kt similarity index 80% rename from plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3WriteSeekableByteChannel.kt rename to src/main/kotlin/app/termora/transfer/s3/S3WriteSeekableByteChannel.kt index ce1f9c6..f2ff55d 100644 --- a/plugins/s3/src/main/kotlin/app/termora/plugins/s3/S3WriteSeekableByteChannel.kt +++ b/src/main/kotlin/app/termora/transfer/s3/S3WriteSeekableByteChannel.kt @@ -1,14 +1,13 @@ -package app.termora.plugins.s3 +package app.termora.transfer.s3 import org.apache.commons.io.IOUtils +import java.io.OutputStream import java.nio.ByteBuffer +import java.nio.channels.Channels import java.nio.channels.SeekableByteChannel -import java.nio.channels.WritableByteChannel - -class S3WriteSeekableByteChannel( - private val channel: WritableByteChannel, -) : SeekableByteChannel { +open class S3WriteSeekableByteChannel(output: OutputStream) : SeekableByteChannel { + private val channel = Channels.newChannel(output) override fun read(dst: ByteBuffer): Int { throw UnsupportedOperationException("read not supported") diff --git a/src/test/kotlin/app/termora/ExposedTest.kt b/src/test/kotlin/app/termora/ExposedTest.kt deleted file mode 100644 index af58cba..0000000 --- a/src/test/kotlin/app/termora/ExposedTest.kt +++ /dev/null @@ -1,39 +0,0 @@ -package app.termora - -import app.termora.database.DataEntity -import app.termora.database.DataType -import app.termora.database.OwnerType -import app.termora.database.SettingEntity -import org.jetbrains.exposed.v1.jdbc.Database -import org.jetbrains.exposed.v1.jdbc.SchemaUtils -import org.jetbrains.exposed.v1.jdbc.insert -import org.jetbrains.exposed.v1.jdbc.transactions.transaction -import kotlin.test.Test - - -class ExposedTest { - - - @Test - fun test() { - val database = Database.connect("jdbc:h2:mem:test", driver = "org.h2.Driver", user = "sa") - - transaction(database) { - SchemaUtils.create(DataEntity, SettingEntity) - - println(DataEntity.insert { - it[ownerId] = "Test" - it[ownerType] = OwnerType.User.name - it[type] = DataType.KeywordHighlight.name - it[data] = "hello 中文".repeat(10000) - } get DataEntity.id) - - println(SettingEntity.insert { - it[name] = "Test" - it[value] = "hello 中文".repeat(10000) - } get SettingEntity.id) - - - } - } -} \ No newline at end of file diff --git a/src/test/kotlin/app/termora/HostTest.kt b/src/test/kotlin/app/termora/HostTest.kt index 1f926ee..c34770d 100644 --- a/src/test/kotlin/app/termora/HostTest.kt +++ b/src/test/kotlin/app/termora/HostTest.kt @@ -10,7 +10,7 @@ class HostTest { """ { "name": "test", - "protocol": SSHProtocolProvider.PROTOCOL, + "protocol": "SSH", "test": "" } """.trimIndent() diff --git a/src/test/kotlin/app/termora/account/ServerManagerTest.kt b/src/test/kotlin/app/termora/account/ServerManagerTest.kt deleted file mode 100644 index 3602219..0000000 --- a/src/test/kotlin/app/termora/account/ServerManagerTest.kt +++ /dev/null @@ -1,15 +0,0 @@ -package app.termora.account - -import kotlin.test.Test - -class ServerManagerTest { - @Test - fun test() { - ServerManager.getInstance().login( - Server( - name = "test", - server = "http://127.0.0.1:8080" - ), "admin", "admin" - ) - } -} \ No newline at end of file diff --git a/src/test/kotlin/app/termora/vfs2/sftp/MySftpFileProviderTest.kt b/src/test/kotlin/app/termora/vfs2/sftp/MySftpFileProviderTest.kt deleted file mode 100644 index 7fc95cd..0000000 --- a/src/test/kotlin/app/termora/vfs2/sftp/MySftpFileProviderTest.kt +++ /dev/null @@ -1,114 +0,0 @@ -package app.termora.vfs2.sftp - -import app.termora.SSHDTest -import app.termora.randomUUID -import org.apache.commons.vfs2.* -import org.apache.commons.vfs2.impl.DefaultFileSystemManager -import org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider -import org.apache.sshd.sftp.client.SftpClientFactory -import java.io.File -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class MySftpFileProviderTest : SSHDTest() { - - companion object { - init { - val fileSystemManager = DefaultFileSystemManager() - fileSystemManager.addProvider("sftp", MySftpFileProvider.instance) - fileSystemManager.addProvider("file", DefaultLocalFileProvider()) - fileSystemManager.init() - VFS.setManager(fileSystemManager) - } - } - - @Test - fun testSetExecutable() { - val file = newFileObject("/config/test.txt") - file.createFile() - file.refresh() - assertFalse(file.isExecutable) - file.setExecutable(true, false) - file.refresh() - assertTrue(file.isExecutable) - } - - @Test - fun testCreateFile() { - val file = newFileObject("/config/test.txt") - assertFalse(file.exists()) - file.createFile() - assertTrue(file.exists()) - } - - @Test - fun testWriteAndReadFile() { - val file = newFileObject("/config/test.txt") - file.createFile() - assertFalse(file.content.isOpen) - - val os = file.content.outputStream - os.write("test".toByteArray()) - os.flush() - assertTrue(file.content.isOpen) - - os.close() - assertFalse(file.content.isOpen) - - val input = file.content.inputStream - assertEquals("test", String(input.readAllBytes())) - assertTrue(file.content.isOpen) - input.close() - assertFalse(file.content.isOpen) - - } - - @Test - fun testCreateFolder() { - val file = newFileObject("/config/test") - assertFalse(file.exists()) - file.createFolder() - assertTrue(file.exists()) - } - - - @Test - fun testSftpClient() { - val session = newClientSession() - val client = SftpClientFactory.instance().createSftpClient(session) - assertTrue(client.isOpen) - session.close() - assertFalse(client.isOpen) - } - - @Test - fun testCopy() { - val file = newFileObject("/config/sshd.pid") - val filepath = File("build", randomUUID()) - val localFile = getVFS().resolveFile("file://${filepath.absolutePath}") - - localFile.copyFrom(file, Selectors.SELECT_ALL) - assertEquals( - file.content.getString(Charsets.UTF_8), - localFile.content.getString(Charsets.UTF_8) - ) - - localFile.delete() - } - - private fun getVFS(): FileSystemManager { - return VFS.getManager() - } - - private fun newFileObject(path: String): FileObject { - val vfs = getVFS() - val fileSystemOptions = FileSystemOptions() - MySftpFileSystemConfigBuilder.getInstance() - .setSftpFileSystem(fileSystemOptions, SftpClientFactory.instance().createSftpFileSystem(newClientSession())) - return vfs.resolveFile("sftp://${path}", fileSystemOptions) - } - - -} \ No newline at end of file