mirror of
https://github.com/TermoraDev/termora.git
synced 2026-01-16 02:12:58 +08:00
feat: support tencent cos
This commit is contained in:
1
.github/workflows/linux-aarch64.yml
vendored
1
.github/workflows/linux-aarch64.yml
vendored
@@ -37,6 +37,7 @@ jobs:
|
||||
# test build
|
||||
- run: |
|
||||
./gradlew build -x test --no-daemon
|
||||
./gradlew clean --no-daemon
|
||||
|
||||
# dist
|
||||
- run: |
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.AuthenticationType
|
||||
import app.termora.Proxy
|
||||
import app.termora.ProxyType
|
||||
import com.qcloud.cos.COSClient
|
||||
import com.qcloud.cos.ClientConfig
|
||||
import com.qcloud.cos.auth.BasicCOSCredentials
|
||||
import com.qcloud.cos.model.Bucket
|
||||
import com.qcloud.cos.region.Region
|
||||
import java.io.Closeable
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
class COSClientHandler(
|
||||
private val cred: BasicCOSCredentials,
|
||||
private val proxy: Proxy,
|
||||
val buckets: List<Bucket>
|
||||
) : Closeable {
|
||||
|
||||
companion object {
|
||||
fun createCOSClient(cred: BasicCOSCredentials, region: String, proxy: Proxy): COSClient {
|
||||
val clientConfig = ClientConfig()
|
||||
if (region.isNotBlank()) {
|
||||
clientConfig.region = Region(region)
|
||||
}
|
||||
clientConfig.isPrintShutdownStackTrace = false
|
||||
if (proxy.type == ProxyType.HTTP) {
|
||||
clientConfig.httpProxyIp = proxy.host
|
||||
clientConfig.httpProxyPort = proxy.port
|
||||
if (proxy.authenticationType == AuthenticationType.Password) {
|
||||
clientConfig.proxyPassword = proxy.password
|
||||
clientConfig.proxyUsername = proxy.username
|
||||
}
|
||||
}
|
||||
return COSClient(cred, clientConfig)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* key: Region
|
||||
* value: Client
|
||||
*/
|
||||
private val clients = mutableMapOf<String, COSClient>()
|
||||
private val closed = AtomicBoolean(false)
|
||||
|
||||
fun getClientForBucket(bucket: String): COSClient {
|
||||
if (closed.get()) throw IllegalStateException("Client already closed")
|
||||
|
||||
synchronized(this) {
|
||||
val bucket = buckets.first { it.name == bucket }
|
||||
if (clients.containsKey(bucket.location)) {
|
||||
return clients.getValue(bucket.location)
|
||||
}
|
||||
clients[bucket.location] = createCOSClient(cred, bucket.location, proxy)
|
||||
return clients.getValue(bucket.location)
|
||||
}
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
synchronized(this) {
|
||||
clients.forEach { it.value.shutdown() }
|
||||
clients.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import org.apache.commons.vfs2.Capability
|
||||
import org.apache.commons.vfs2.FileName
|
||||
import org.apache.commons.vfs2.FileSystem
|
||||
import org.apache.commons.vfs2.FileSystemOptions
|
||||
import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider
|
||||
|
||||
class COSFileProvider private constructor() : AbstractOriginatingFileProvider() {
|
||||
|
||||
companion object {
|
||||
val instance by lazy { COSFileProvider() }
|
||||
val capabilities = listOf(
|
||||
Capability.CREATE,
|
||||
Capability.DELETE,
|
||||
Capability.RENAME,
|
||||
Capability.GET_TYPE,
|
||||
Capability.LIST_CHILDREN,
|
||||
Capability.READ_CONTENT,
|
||||
Capability.URI,
|
||||
Capability.WRITE_CONTENT,
|
||||
Capability.GET_LAST_MODIFIED,
|
||||
Capability.SET_LAST_MODIFIED_FILE,
|
||||
Capability.RANDOM_ACCESS_READ,
|
||||
Capability.APPEND_CONTENT
|
||||
)
|
||||
}
|
||||
|
||||
override fun getCapabilities(): Collection<Capability> {
|
||||
return COSFileProvider.capabilities
|
||||
}
|
||||
|
||||
override fun doCreateFileSystem(
|
||||
rootFileName: FileName,
|
||||
fileSystemOptions: FileSystemOptions
|
||||
): FileSystem? {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.transfer.s3.S3FileSystem
|
||||
import org.apache.commons.io.IOUtils
|
||||
|
||||
/**
|
||||
* key: region
|
||||
*/
|
||||
class COSFileSystem(private val clientHandler: COSClientHandler) :
|
||||
S3FileSystem(COSFileSystemProvider(clientHandler)) {
|
||||
|
||||
override fun close() {
|
||||
IOUtils.closeQuietly(clientHandler)
|
||||
super.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.transfer.s3.S3FileAttributes
|
||||
import app.termora.transfer.s3.S3FileSystemProvider
|
||||
import app.termora.transfer.s3.S3Path
|
||||
import com.qcloud.cos.model.ListObjectsRequest
|
||||
import com.qcloud.cos.model.ObjectMetadata
|
||||
import com.qcloud.cos.model.PutObjectRequest
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import kotlin.io.path.absolutePathString
|
||||
|
||||
class COSFileSystemProvider(private val clientHandler: COSClientHandler) : S3FileSystemProvider() {
|
||||
|
||||
|
||||
override fun getScheme(): String? {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
override fun getOutputStream(path: S3Path): OutputStream {
|
||||
return createStreamer(path)
|
||||
}
|
||||
|
||||
override fun getInputStream(path: S3Path): InputStream {
|
||||
val client = clientHandler.getClientForBucket(path.bucketName)
|
||||
return client.getObject(path.bucketName, path.objectName).objectContent
|
||||
}
|
||||
|
||||
private fun createStreamer(path: S3Path): OutputStream {
|
||||
val pis = PipedInputStream()
|
||||
val pos = PipedOutputStream(pis)
|
||||
val exception = AtomicReference<Throwable>()
|
||||
|
||||
val thread = Thread.ofVirtual().start {
|
||||
try {
|
||||
val client = clientHandler.getClientForBucket(path.bucketName)
|
||||
client.putObject(PutObjectRequest(path.bucketName, path.objectName, pis, ObjectMetadata()))
|
||||
} catch (e: Exception) {
|
||||
exception.set(e)
|
||||
} finally {
|
||||
IOUtils.closeQuietly(pis)
|
||||
}
|
||||
}
|
||||
|
||||
return object : OutputStream() {
|
||||
override fun write(b: Int) {
|
||||
val exception = exception.get()
|
||||
if (exception != null) throw exception
|
||||
pos.write(b)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
pos.close()
|
||||
if (thread.isAlive) thread.join()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun fetchChildren(path: S3Path): MutableList<S3Path> {
|
||||
val paths = mutableListOf<S3Path>()
|
||||
|
||||
// root
|
||||
if (path.isRoot) {
|
||||
for (bucket in clientHandler.buckets) {
|
||||
val p = path.resolve(bucket.name)
|
||||
p.attributes = S3FileAttributes(
|
||||
directory = true,
|
||||
lastModifiedTime = bucket.creationDate.toInstant().toEpochMilli()
|
||||
)
|
||||
paths.add(p)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
var nextMarker = StringUtils.EMPTY
|
||||
val maxKeys = 100
|
||||
val bucketName = path.bucketName
|
||||
while (true) {
|
||||
val request = ListObjectsRequest()
|
||||
.withBucketName(bucketName)
|
||||
.withMaxKeys(maxKeys)
|
||||
.withDelimiter(path.fileSystem.separator)
|
||||
|
||||
if (path.objectName.isNotBlank()) request.withPrefix(path.objectName + path.fileSystem.separator)
|
||||
if (nextMarker.isNotBlank()) request.withMarker(nextMarker)
|
||||
|
||||
|
||||
val objectListing = clientHandler.getClientForBucket(bucketName).listObjects(request)
|
||||
for (e in objectListing.commonPrefixes) {
|
||||
val p = path.bucket.resolve(e)
|
||||
p.attributes = p.attributes.copy(directory = true)
|
||||
delete(p)
|
||||
paths.add(p)
|
||||
}
|
||||
|
||||
for (e in objectListing.objectSummaries) {
|
||||
val p = path.bucket.resolve(e.key)
|
||||
p.attributes = p.attributes.copy(
|
||||
regularFile = true, size = e.size,
|
||||
lastModifiedTime = e.lastModified.time
|
||||
)
|
||||
paths.add(p)
|
||||
}
|
||||
|
||||
if (objectListing.isTruncated.not()) {
|
||||
break
|
||||
}
|
||||
|
||||
nextMarker = objectListing.nextMarker
|
||||
|
||||
}
|
||||
|
||||
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
override fun delete(path: S3Path, isDirectory: Boolean) {
|
||||
if (isDirectory.not())
|
||||
clientHandler.getClientForBucket(path.bucketName).deleteObject(path.bucketName, path.objectName)
|
||||
}
|
||||
|
||||
override fun checkAccess(path: S3Path, vararg modes: AccessMode) {
|
||||
try {
|
||||
val client = clientHandler.getClientForBucket(path.bucketName)
|
||||
if (client.doesObjectExist(path.bucketName, path.objectName).not()) {
|
||||
throw NoSuchFileException(path.objectName)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (e is NoSuchFileException) throw e
|
||||
throw NoSuchFileException(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.*
|
||||
import app.termora.plugin.internal.BasicProxyOption
|
||||
import com.formdev.flatlaf.FlatClientProperties
|
||||
import com.formdev.flatlaf.ui.FlatTextBorder
|
||||
import com.jgoodies.forms.builder.FormBuilder
|
||||
import com.jgoodies.forms.layout.FormLayout
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import java.awt.BorderLayout
|
||||
import java.awt.KeyboardFocusManager
|
||||
import java.awt.event.ComponentAdapter
|
||||
import java.awt.event.ComponentEvent
|
||||
import javax.swing.*
|
||||
|
||||
class COSHostOptionsPane : OptionsPane() {
|
||||
private val generalOption = GeneralOption()
|
||||
private val proxyOption = BasicProxyOption(listOf(ProxyType.HTTP))
|
||||
private val sftpOption = SFTPOption()
|
||||
|
||||
init {
|
||||
addOption(generalOption)
|
||||
addOption(proxyOption)
|
||||
addOption(sftpOption)
|
||||
|
||||
}
|
||||
|
||||
fun getHost(): Host {
|
||||
val name = generalOption.nameTextField.text
|
||||
val protocol = COSProtocolProvider.PROTOCOL
|
||||
val port = 0
|
||||
var authentication = Authentication.Companion.No
|
||||
var proxy = Proxy.Companion.No
|
||||
val authenticationType = AuthenticationType.Password
|
||||
|
||||
authentication = authentication.copy(
|
||||
type = authenticationType,
|
||||
password = String(generalOption.passwordTextField.password)
|
||||
)
|
||||
|
||||
|
||||
if (proxyOption.proxyTypeComboBox.selectedItem != ProxyType.No) {
|
||||
proxy = proxy.copy(
|
||||
type = proxyOption.proxyTypeComboBox.selectedItem as ProxyType,
|
||||
host = proxyOption.proxyHostTextField.text,
|
||||
username = proxyOption.proxyUsernameTextField.text,
|
||||
password = String(proxyOption.proxyPasswordTextField.password),
|
||||
port = proxyOption.proxyPortTextField.value as Int,
|
||||
authenticationType = proxyOption.proxyAuthenticationTypeComboBox.selectedItem as AuthenticationType,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
val options = Options.Default.copy(
|
||||
sftpDefaultDirectory = sftpOption.defaultDirectoryField.text,
|
||||
extras = mutableMapOf(
|
||||
"cos.delimiter" to generalOption.delimiterTextField.text,
|
||||
)
|
||||
)
|
||||
|
||||
return Host(
|
||||
name = name,
|
||||
protocol = protocol,
|
||||
port = port,
|
||||
username = generalOption.usernameTextField.text,
|
||||
authentication = authentication,
|
||||
proxy = proxy,
|
||||
sort = System.currentTimeMillis(),
|
||||
remark = generalOption.remarkTextArea.text,
|
||||
options = options,
|
||||
)
|
||||
}
|
||||
|
||||
fun setHost(host: Host) {
|
||||
generalOption.nameTextField.text = host.name
|
||||
generalOption.usernameTextField.text = host.username
|
||||
generalOption.remarkTextArea.text = host.remark
|
||||
generalOption.passwordTextField.text = host.authentication.password
|
||||
generalOption.delimiterTextField.text = host.options.extras["cos.delimiter"] ?: StringUtils.EMPTY
|
||||
|
||||
proxyOption.proxyTypeComboBox.selectedItem = host.proxy.type
|
||||
proxyOption.proxyHostTextField.text = host.proxy.host
|
||||
proxyOption.proxyPasswordTextField.text = host.proxy.password
|
||||
proxyOption.proxyUsernameTextField.text = host.proxy.username
|
||||
proxyOption.proxyPortTextField.value = host.proxy.port
|
||||
proxyOption.proxyAuthenticationTypeComboBox.selectedItem = host.proxy.authenticationType
|
||||
|
||||
|
||||
sftpOption.defaultDirectoryField.text = host.options.sftpDefaultDirectory
|
||||
}
|
||||
|
||||
fun validateFields(): Boolean {
|
||||
val host = getHost()
|
||||
|
||||
// general
|
||||
if (validateField(generalOption.nameTextField)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (validateField(generalOption.usernameTextField)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (host.authentication.type == AuthenticationType.Password) {
|
||||
if (validateField(generalOption.passwordTextField)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// proxy
|
||||
if (host.proxy.type != ProxyType.No) {
|
||||
if (validateField(proxyOption.proxyHostTextField)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (host.proxy.authenticationType != AuthenticationType.No) {
|
||||
if (validateField(proxyOption.proxyUsernameTextField)
|
||||
|| validateField(proxyOption.proxyPasswordTextField)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回 true 表示有错误
|
||||
*/
|
||||
private fun validateField(textField: JTextField): Boolean {
|
||||
if (textField.isEnabled && textField.text.isBlank()) {
|
||||
setOutlineError(textField)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private fun setOutlineError(c: JComponent) {
|
||||
selectOptionJComponent(c)
|
||||
c.putClientProperty(FlatClientProperties.OUTLINE, FlatClientProperties.OUTLINE_ERROR)
|
||||
c.requestFocusInWindow()
|
||||
}
|
||||
|
||||
|
||||
private inner class GeneralOption : JPanel(BorderLayout()), Option {
|
||||
val nameTextField = OutlineTextField(128)
|
||||
val usernameTextField = OutlineTextField(128)
|
||||
val passwordTextField = OutlinePasswordField(255)
|
||||
val remarkTextArea = FixedLengthTextArea(512)
|
||||
|
||||
// val regionComboBox = OutlineComboBox<String>()
|
||||
val delimiterTextField = OutlineTextField(128)
|
||||
|
||||
init {
|
||||
initView()
|
||||
initEvents()
|
||||
}
|
||||
|
||||
private fun initView() {
|
||||
|
||||
/*regionComboBox.addItem("ap-beijing-1")
|
||||
regionComboBox.addItem("ap-beijing")
|
||||
regionComboBox.addItem("ap-nanjing")
|
||||
regionComboBox.addItem("ap-shanghai")
|
||||
regionComboBox.addItem("ap-guangzhou")
|
||||
regionComboBox.addItem("ap-chengdu")
|
||||
regionComboBox.addItem("ap-chongqing")
|
||||
regionComboBox.addItem("ap-shenzhen-fsi")
|
||||
regionComboBox.addItem("ap-shanghai-fsi")
|
||||
regionComboBox.addItem("ap-beijing-fsi")
|
||||
|
||||
regionComboBox.addItem("ap-hongkong")
|
||||
regionComboBox.addItem("ap-singapore")
|
||||
regionComboBox.addItem("ap-jakarta")
|
||||
regionComboBox.addItem("ap-seoul")
|
||||
regionComboBox.addItem("ap-bangkok")
|
||||
regionComboBox.addItem("ap-tokyo")
|
||||
regionComboBox.addItem("na-siliconvalley")
|
||||
regionComboBox.addItem("na-ashburn")
|
||||
regionComboBox.addItem("sa-saopaulo")
|
||||
regionComboBox.addItem("eu-frankfurt")
|
||||
|
||||
regionComboBox.isEditable = true*/
|
||||
|
||||
delimiterTextField.text = "/"
|
||||
delimiterTextField.isEditable = false
|
||||
add(getCenterComponent(), BorderLayout.CENTER)
|
||||
}
|
||||
|
||||
private fun initEvents() {
|
||||
|
||||
addComponentListener(object : ComponentAdapter() {
|
||||
override fun componentResized(e: ComponentEvent) {
|
||||
SwingUtilities.invokeLater { nameTextField.requestFocusInWindow() }
|
||||
removeComponentListener(this)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
override fun getIcon(isSelected: Boolean): Icon {
|
||||
return Icons.settings
|
||||
}
|
||||
|
||||
override fun getTitle(): String {
|
||||
return I18n.getString("termora.new-host.general")
|
||||
}
|
||||
|
||||
override fun getJComponent(): JComponent {
|
||||
return this
|
||||
}
|
||||
|
||||
private fun getCenterComponent(): JComponent {
|
||||
val layout = FormLayout(
|
||||
"left:pref, $FORM_MARGIN, default:grow, $FORM_MARGIN, pref, $FORM_MARGIN, default",
|
||||
"pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref"
|
||||
)
|
||||
remarkTextArea.setFocusTraversalKeys(
|
||||
KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS,
|
||||
KeyboardFocusManager.getCurrentKeyboardFocusManager()
|
||||
.getDefaultFocusTraversalKeys(KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS)
|
||||
)
|
||||
remarkTextArea.setFocusTraversalKeys(
|
||||
KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS,
|
||||
KeyboardFocusManager.getCurrentKeyboardFocusManager()
|
||||
.getDefaultFocusTraversalKeys(KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS)
|
||||
)
|
||||
|
||||
remarkTextArea.rows = 8
|
||||
remarkTextArea.lineWrap = true
|
||||
remarkTextArea.border = BorderFactory.createEmptyBorder(4, 4, 4, 4)
|
||||
|
||||
|
||||
var rows = 1
|
||||
val step = 2
|
||||
val panel = FormBuilder.create().layout(layout)
|
||||
.add("${I18n.getString("termora.new-host.general.name")}:").xy(1, rows)
|
||||
.add(nameTextField).xyw(3, rows, 5).apply { rows += step }
|
||||
|
||||
.add("SecretId:").xy(1, rows)
|
||||
.add(usernameTextField).xyw(3, rows, 5).apply { rows += step }
|
||||
|
||||
.add("SecretKey:").xy(1, rows)
|
||||
.add(passwordTextField).xyw(3, rows, 5).apply { rows += step }
|
||||
|
||||
.add("Delimiter:").xy(1, rows)
|
||||
.add(delimiterTextField).xyw(3, rows, 5).apply { rows += step }
|
||||
|
||||
.add("${I18n.getString("termora.new-host.general.remark")}:").xy(1, rows)
|
||||
.add(JScrollPane(remarkTextArea).apply { border = FlatTextBorder() })
|
||||
.xyw(3, rows, 5).apply { rows += step }
|
||||
|
||||
.build()
|
||||
|
||||
|
||||
return panel
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private inner class SFTPOption : JPanel(BorderLayout()), Option {
|
||||
val defaultDirectoryField = OutlineTextField(255)
|
||||
|
||||
|
||||
init {
|
||||
initView()
|
||||
initEvents()
|
||||
}
|
||||
|
||||
private fun initView() {
|
||||
add(getCenterComponent(), BorderLayout.CENTER)
|
||||
}
|
||||
|
||||
private fun initEvents() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
override fun getIcon(isSelected: Boolean): Icon {
|
||||
return Icons.folder
|
||||
}
|
||||
|
||||
override fun getTitle(): String {
|
||||
return I18n.getString("termora.transport.sftp")
|
||||
}
|
||||
|
||||
override fun getJComponent(): JComponent {
|
||||
return this
|
||||
}
|
||||
|
||||
private fun getCenterComponent(): JComponent {
|
||||
val layout = FormLayout(
|
||||
"left:pref, $FORM_MARGIN, default:grow",
|
||||
"pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref, $FORM_MARGIN, pref"
|
||||
)
|
||||
|
||||
var rows = 1
|
||||
val step = 2
|
||||
val panel = FormBuilder.create().layout(layout)
|
||||
.add("${I18n.getString("termora.settings.sftp.default-directory")}:").xy(1, rows)
|
||||
.add(defaultDirectoryField).xy(3, rows).apply { rows += step }
|
||||
.build()
|
||||
|
||||
|
||||
return panel
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,8 +1,5 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.DynamicIcon
|
||||
import app.termora.I18n
|
||||
import app.termora.Icons
|
||||
import app.termora.plugin.Extension
|
||||
import app.termora.plugin.ExtensionSupport
|
||||
import app.termora.plugin.PaidPlugin
|
||||
@@ -13,8 +10,8 @@ class COSPlugin : PaidPlugin {
|
||||
private val support = ExtensionSupport()
|
||||
|
||||
init {
|
||||
support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.Companion.instance }
|
||||
support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.Companion.instance }
|
||||
support.addExtension(ProtocolProviderExtension::class.java) { COSProtocolProviderExtension.instance }
|
||||
support.addExtension(ProtocolHostPanelExtension::class.java) { COSProtocolHostPanelExtension.instance }
|
||||
}
|
||||
|
||||
override fun getAuthor(): String {
|
||||
|
||||
@@ -1,22 +1,36 @@
|
||||
package app.termora.plugins.cos
|
||||
|
||||
import app.termora.Disposer
|
||||
import app.termora.Host
|
||||
import app.termora.protocol.ProtocolHostPanel
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import java.awt.BorderLayout
|
||||
|
||||
class COSProtocolHostPanel : ProtocolHostPanel() {
|
||||
|
||||
private val pane = COSHostOptionsPane()
|
||||
|
||||
init {
|
||||
initView()
|
||||
initEvents()
|
||||
}
|
||||
|
||||
|
||||
private fun initView() {
|
||||
add(pane, BorderLayout.CENTER)
|
||||
Disposer.register(this, pane)
|
||||
}
|
||||
|
||||
private fun initEvents() {}
|
||||
|
||||
override fun getHost(): Host {
|
||||
return Host(
|
||||
name = StringUtils.EMPTY,
|
||||
protocol = COSProtocolProvider.Companion.PROTOCOL
|
||||
)
|
||||
return pane.getHost()
|
||||
}
|
||||
|
||||
override fun setHost(host: Host) {
|
||||
|
||||
pane.setHost(host)
|
||||
}
|
||||
|
||||
override fun validateFields(): Boolean {
|
||||
return true
|
||||
return pane.validateFields()
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ class COSProtocolHostPanelExtension private constructor() : ProtocolHostPanelExt
|
||||
}
|
||||
|
||||
override fun getProtocolProvider(): ProtocolProvider {
|
||||
return COSProtocolProvider.Companion.instance
|
||||
return COSProtocolProvider.instance
|
||||
}
|
||||
|
||||
override fun createProtocolHostPanel(): ProtocolHostPanel {
|
||||
|
||||
@@ -2,10 +2,14 @@ package app.termora.plugins.cos
|
||||
|
||||
import app.termora.DynamicIcon
|
||||
import app.termora.Icons
|
||||
import app.termora.protocol.FileObjectHandler
|
||||
import app.termora.protocol.FileObjectRequest
|
||||
import app.termora.protocol.PathHandler
|
||||
import app.termora.protocol.PathHandlerRequest
|
||||
import app.termora.protocol.TransferProtocolProvider
|
||||
import org.apache.commons.vfs2.provider.FileProvider
|
||||
import com.qcloud.cos.ClientConfig
|
||||
import com.qcloud.cos.auth.BasicCOSCredentials
|
||||
import com.qcloud.cos.model.Bucket
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
|
||||
|
||||
class COSProtocolProvider private constructor() : TransferProtocolProvider {
|
||||
|
||||
@@ -22,12 +26,30 @@ class COSProtocolProvider private constructor() : TransferProtocolProvider {
|
||||
return Icons.tencent
|
||||
}
|
||||
|
||||
override fun getFileProvider(): FileProvider {
|
||||
return COSFileProvider.instance
|
||||
override fun createPathHandler(requester: PathHandlerRequest): PathHandler {
|
||||
val host = requester.host
|
||||
val secretId = host.username
|
||||
val secretKey = host.authentication.password
|
||||
val cred = BasicCOSCredentials(secretId, secretKey)
|
||||
val clientConfig = ClientConfig()
|
||||
|
||||
clientConfig.isPrintShutdownStackTrace = false
|
||||
val cosClient = COSClientHandler.createCOSClient(cred, StringUtils.EMPTY, host.proxy)
|
||||
val buckets: List<Bucket>
|
||||
|
||||
try {
|
||||
buckets = cosClient.listBuckets()
|
||||
if (buckets.isEmpty()) {
|
||||
throw IllegalStateException("没有获取到桶信息")
|
||||
}
|
||||
} finally {
|
||||
cosClient.shutdown()
|
||||
}
|
||||
|
||||
override fun getRootFileObject(requester: FileObjectRequest): FileObjectHandler {
|
||||
TODO("Not yet implemented")
|
||||
val defaultPath = host.options.sftpDefaultDirectory
|
||||
val fs = COSFileSystem(COSClientHandler(cred, host.proxy, buckets))
|
||||
return PathHandler(fs, fs.getPath(defaultPath))
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -9,6 +9,6 @@ class COSProtocolProviderExtension private constructor() : ProtocolProviderExten
|
||||
}
|
||||
|
||||
override fun getProtocolProvider(): ProtocolProvider {
|
||||
return COSProtocolProvider.Companion.instance
|
||||
return COSProtocolProvider.instance
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ plugins {
|
||||
}
|
||||
|
||||
|
||||
project.version = "0.0.4"
|
||||
project.version = "0.0.5"
|
||||
|
||||
|
||||
dependencies {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package app.termora.plugins.s3
|
||||
|
||||
import app.termora.transfer.s3.S3FileSystem
|
||||
import io.minio.MinioClient
|
||||
|
||||
class MyS3FileSystem(private val minioClient: MinioClient) :
|
||||
S3FileSystem(MyS3FileSystemProvider(minioClient)) {
|
||||
|
||||
override fun close() {
|
||||
minioClient.close()
|
||||
super.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package app.termora.plugins.s3
|
||||
|
||||
import app.termora.transfer.s3.S3FileAttributes
|
||||
import app.termora.transfer.s3.S3FileSystemProvider
|
||||
import app.termora.transfer.s3.S3Path
|
||||
import io.minio.*
|
||||
import io.minio.errors.ErrorResponseException
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
import java.nio.file.AccessMode
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import kotlin.io.path.absolutePathString
|
||||
|
||||
class MyS3FileSystemProvider(private val minioClient: MinioClient) : S3FileSystemProvider() {
|
||||
|
||||
override fun getScheme(): String? {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
override fun getOutputStream(path: S3Path): OutputStream {
|
||||
return createStreamer(path)
|
||||
}
|
||||
|
||||
override fun getInputStream(path: S3Path): InputStream {
|
||||
return minioClient.getObject(
|
||||
GetObjectArgs.builder().bucket(path.bucketName)
|
||||
.`object`(path.objectName).build()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
private fun createStreamer(path: S3Path): OutputStream {
|
||||
val pis = PipedInputStream()
|
||||
val pos = PipedOutputStream(pis)
|
||||
val exception = AtomicReference<Throwable>()
|
||||
|
||||
val thread = Thread.ofVirtual().start {
|
||||
try {
|
||||
minioClient.putObject(
|
||||
PutObjectArgs.builder()
|
||||
.bucket(path.bucketName)
|
||||
.stream(pis, -1, 32 * 1024 * 1024)
|
||||
.`object`(path.objectName).build()
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
exception.set(e)
|
||||
} finally {
|
||||
IOUtils.closeQuietly(pis)
|
||||
}
|
||||
}
|
||||
|
||||
return object : OutputStream() {
|
||||
override fun write(b: Int) {
|
||||
val exception = exception.get()
|
||||
if (exception != null) throw exception
|
||||
pos.write(b)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
pos.close()
|
||||
if (thread.isAlive) thread.join()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun fetchChildren(path: S3Path): MutableList<S3Path> {
|
||||
val paths = mutableListOf<S3Path>()
|
||||
|
||||
// root
|
||||
if (path.isRoot) {
|
||||
for (bucket in minioClient.listBuckets()) {
|
||||
val p = path.resolve(bucket.name())
|
||||
p.attributes = S3FileAttributes(
|
||||
directory = true,
|
||||
lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
|
||||
)
|
||||
paths.add(p)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
var startAfter = StringUtils.EMPTY
|
||||
val maxKeys = 100
|
||||
|
||||
while (true) {
|
||||
val builder = ListObjectsArgs.builder()
|
||||
.bucket(path.bucketName)
|
||||
.maxKeys(maxKeys)
|
||||
.delimiter(path.fileSystem.separator)
|
||||
|
||||
if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
|
||||
if (startAfter.isNotBlank()) builder.startAfter(startAfter)
|
||||
|
||||
|
||||
val subPaths = mutableListOf<S3Path>()
|
||||
for (e in minioClient.listObjects(builder.build())) {
|
||||
val item = e.get()
|
||||
val p = path.bucket.resolve(item.objectName())
|
||||
var attributes = p.attributes.copy(
|
||||
directory = item.isDir,
|
||||
regularFile = item.isDir.not(),
|
||||
size = item.size()
|
||||
)
|
||||
if (item.lastModified() != null) {
|
||||
attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
|
||||
}
|
||||
p.attributes = attributes
|
||||
|
||||
// 如果是文件夹,那么就要删除内存中的
|
||||
if (attributes.isDirectory) {
|
||||
delete(p)
|
||||
}
|
||||
|
||||
subPaths.add(p)
|
||||
startAfter = item.objectName()
|
||||
}
|
||||
|
||||
paths.addAll(subPaths)
|
||||
|
||||
if (subPaths.size < maxKeys)
|
||||
break
|
||||
|
||||
|
||||
}
|
||||
|
||||
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
|
||||
override fun delete(path: S3Path, isDirectory: Boolean) {
|
||||
if (isDirectory.not())
|
||||
minioClient.removeObject(
|
||||
RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
override fun checkAccess(path: S3Path, vararg modes: AccessMode) {
|
||||
try {
|
||||
minioClient.statObject(
|
||||
StatObjectArgs.builder()
|
||||
.`object`(path.objectName)
|
||||
.bucket(path.bucketName).build()
|
||||
)
|
||||
} catch (e: ErrorResponseException) {
|
||||
throw NoSuchFileException(e.errorResponse().message())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -70,7 +70,7 @@ class S3ProtocolProvider private constructor() : TransferProtocolProvider {
|
||||
// val delimiter = host.options.extras["s3.delimiter"] ?: "/"
|
||||
val defaultPath = host.options.sftpDefaultDirectory
|
||||
val minioClient = builder.build()
|
||||
val fs = S3FileSystem(minioClient)
|
||||
val fs = MyS3FileSystem(minioClient)
|
||||
return PathHandler(fs, fs.getPath(defaultPath))
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ class S3FileSystemTest {
|
||||
}
|
||||
}
|
||||
|
||||
val fileSystem = S3FileSystem(minioClient)
|
||||
val fileSystem = MyS3FileSystem(minioClient)
|
||||
val path = fileSystem.getPath("/")
|
||||
PathWalker.walkFileTree(path, object : PathVisitor {
|
||||
override fun preVisitDirectory(
|
||||
|
||||
@@ -5,7 +5,7 @@ rootProject.name = "termora"
|
||||
|
||||
include("plugins:s3")
|
||||
//include("plugins:oss")
|
||||
//include("plugins:cos")
|
||||
include("plugins:cos")
|
||||
//include("plugins:obs")
|
||||
//include("plugins:ftp")
|
||||
include("plugins:bg")
|
||||
|
||||
@@ -10,7 +10,8 @@ import java.awt.Component
|
||||
import java.awt.event.ItemEvent
|
||||
import javax.swing.*
|
||||
|
||||
class BasicProxyOption : JPanel(BorderLayout()), Option {
|
||||
class BasicProxyOption(private val proxyTypes: List<ProxyType> = listOf(ProxyType.HTTP, ProxyType.SOCKS5)) :
|
||||
JPanel(BorderLayout()), Option {
|
||||
private val formMargin = "7dlu"
|
||||
|
||||
val proxyTypeComboBox = FlatComboBox<ProxyType>()
|
||||
@@ -61,8 +62,9 @@ class BasicProxyOption : JPanel(BorderLayout()), Option {
|
||||
}
|
||||
|
||||
proxyTypeComboBox.addItem(ProxyType.No)
|
||||
proxyTypeComboBox.addItem(ProxyType.HTTP)
|
||||
proxyTypeComboBox.addItem(ProxyType.SOCKS5)
|
||||
for (type in proxyTypes) {
|
||||
proxyTypeComboBox.addItem(type)
|
||||
}
|
||||
|
||||
proxyAuthenticationTypeComboBox.addItem(AuthenticationType.No)
|
||||
proxyAuthenticationTypeComboBox.addItem(AuthenticationType.Password)
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package app.termora.protocol
|
||||
|
||||
import app.termora.Disposable
|
||||
import org.apache.commons.io.IOUtils
|
||||
import java.nio.file.FileSystem
|
||||
import java.nio.file.Path
|
||||
|
||||
open class PathHandler(val fileSystem: FileSystem, val path: Path) : Disposable {
|
||||
|
||||
override fun dispose() {
|
||||
IOUtils.closeQuietly(fileSystem)
|
||||
}
|
||||
}
|
||||
@@ -442,11 +442,6 @@ class TransferVisualWindow(tab: SSHTerminalTab, visualWindowManager: VisualWindo
|
||||
override fun getWorkdir(): Path? {
|
||||
return transportPanel.workdir
|
||||
}
|
||||
|
||||
override fun getTableModel(): TransportTableModel? {
|
||||
return transportPanel.getTableModel()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,7 +452,6 @@ class TransferVisualWindow(tab: SSHTerminalTab, visualWindowManager: VisualWindo
|
||||
transferManager,
|
||||
object : DefaultInternalTransferManager.WorkdirProvider {
|
||||
override fun getWorkdir() = null
|
||||
override fun getTableModel() = null
|
||||
},
|
||||
createWorkdirProvider(transportPanel)
|
||||
)
|
||||
|
||||
@@ -31,6 +31,7 @@ import kotlin.collections.ArrayDeque
|
||||
import kotlin.collections.List
|
||||
import kotlin.collections.Set
|
||||
import kotlin.collections.isNotEmpty
|
||||
import kotlin.io.path.exists
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.pathString
|
||||
import kotlin.math.max
|
||||
@@ -49,7 +50,6 @@ class DefaultInternalTransferManager(
|
||||
|
||||
interface WorkdirProvider {
|
||||
fun getWorkdir(): Path?
|
||||
fun getTableModel(): TransportTableModel?
|
||||
}
|
||||
|
||||
|
||||
@@ -85,19 +85,14 @@ class DefaultInternalTransferManager(
|
||||
if (paths.isEmpty()) return CompletableFuture.completedFuture(Unit)
|
||||
|
||||
val future = CompletableFuture<Unit>()
|
||||
val tableModel = when (targetWorkdir.fileSystem) {
|
||||
source.getWorkdir()?.fileSystem -> source.getTableModel()
|
||||
target.getWorkdir()?.fileSystem -> target.getTableModel()
|
||||
else -> null
|
||||
}
|
||||
|
||||
coroutineScope.launch(Dispatchers.IO) {
|
||||
try {
|
||||
val context = AskTransferContext(TransferAction.Overwrite, false)
|
||||
for (pair in paths) {
|
||||
if (mode == TransferMode.Transfer && tableModel != null && context.applyAll.not()) {
|
||||
if (mode == TransferMode.Transfer && context.applyAll.not()) {
|
||||
val action = withContext(Dispatchers.Swing) {
|
||||
getTransferAction(context, tableModel, pair.second)
|
||||
getTransferAction(context, targetWorkdir.resolve(pair.first.name), pair.second)
|
||||
}
|
||||
if (action == null) {
|
||||
break
|
||||
@@ -143,21 +138,19 @@ class DefaultInternalTransferManager(
|
||||
|
||||
private fun getTransferAction(
|
||||
context: AskTransferContext,
|
||||
model: TransportTableModel,
|
||||
path: Path,
|
||||
source: TransportTableModel.Attributes
|
||||
): TransferAction? {
|
||||
if (context.applyAll) return context.action
|
||||
|
||||
for (i in 0 until model.rowCount) {
|
||||
val c = model.getAttributes(i)
|
||||
if (c.name != source.name) continue
|
||||
val transfer = askTransfer(source, c)
|
||||
if (path.exists()) {
|
||||
val transfer = askTransfer(source, source)
|
||||
context.action = transfer.action
|
||||
context.applyAll = transfer.applyAll
|
||||
if (transfer.option != JOptionPane.OK_OPTION) return null
|
||||
}
|
||||
|
||||
return context.action
|
||||
return TransferAction.Overwrite
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardOpenOption
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlin.io.path.inputStream
|
||||
import kotlin.io.path.outputStream
|
||||
|
||||
@@ -18,8 +19,12 @@ class FileTransfer(
|
||||
private lateinit var input: InputStream
|
||||
private lateinit var output: OutputStream
|
||||
|
||||
private val closed = AtomicBoolean(false)
|
||||
|
||||
override suspend fun transfer(bufferSize: Int): Long {
|
||||
|
||||
if (closed.get()) throw IllegalStateException("Transfer already closed")
|
||||
|
||||
if (::input.isInitialized.not()) {
|
||||
input = source().inputStream(StandardOpenOption.READ)
|
||||
}
|
||||
@@ -48,6 +53,7 @@ class FileTransfer(
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
if (::input.isInitialized) {
|
||||
IOUtils.closeQuietly(input)
|
||||
}
|
||||
@@ -56,5 +62,6 @@ class FileTransfer(
|
||||
IOUtils.closeQuietly(output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -424,6 +424,11 @@ class TransferTableModel(private val coroutineScope: CoroutineScope) :
|
||||
// 异步上报,因为数据量非常大,所以采用异步
|
||||
reporter.report(node, len, System.currentTimeMillis())
|
||||
}
|
||||
|
||||
// 因为可能是异步传输,只有关闭后才能确保数据已经到达云端
|
||||
// 尤其是 S3 协议
|
||||
if (transfer is Closeable) IOUtils.closeQuietly(transfer)
|
||||
|
||||
withContext(Dispatchers.Swing) {
|
||||
if (continueTransfer(node)) {
|
||||
changeState(node, State.Done)
|
||||
|
||||
@@ -290,7 +290,7 @@ class TransportPanel(
|
||||
// 传输完成之后刷新
|
||||
transferManager.addTransferListener(object : TransferListener {
|
||||
override fun onTransferChanged(transfer: Transfer, state: TransferTreeTableNode.State) {
|
||||
if (state != TransferTreeTableNode.State.Done) return
|
||||
if (state != TransferTreeTableNode.State.Done && state != TransferTreeTableNode.State.Failed) return
|
||||
if (transfer.target().fileSystem != _fileSystem) return
|
||||
if (transfer.target() == workdir || transfer.target().parent == workdir) {
|
||||
reload(requestFocus = false)
|
||||
|
||||
@@ -95,9 +95,6 @@ class TransportViewer : JPanel(BorderLayout()), DataProvider, Disposable {
|
||||
return source.getSelectedTransportPanel()?.workdir
|
||||
}
|
||||
|
||||
override fun getTableModel(): TransportTableModel? {
|
||||
return source.getSelectedTransportPanel()?.getTableModel()
|
||||
}
|
||||
|
||||
},
|
||||
object : DefaultInternalTransferManager.WorkdirProvider {
|
||||
@@ -105,9 +102,6 @@ class TransportViewer : JPanel(BorderLayout()), DataProvider, Disposable {
|
||||
return target.getSelectedTransportPanel()?.workdir
|
||||
}
|
||||
|
||||
override fun getTableModel(): TransportTableModel? {
|
||||
return target.getSelectedTransportPanel()?.getTableModel()
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
@@ -24,8 +24,8 @@ internal class SFTPPathHandler(
|
||||
}
|
||||
|
||||
override fun dispose() {
|
||||
super.dispose()
|
||||
session.removeCloseFutureListener(listener)
|
||||
IOUtils.closeQuietly(fileSystem)
|
||||
IOUtils.closeQuietly(session)
|
||||
IOUtils.closeQuietly(client)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileTime
|
||||
@@ -1,14 +1,11 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import io.minio.MinioClient
|
||||
import org.apache.sshd.common.file.util.BaseFileSystem
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.attribute.UserPrincipalLookupService
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
class S3FileSystem(
|
||||
private val minioClient: MinioClient,
|
||||
) : BaseFileSystem<S3Path>(S3FileSystemProvider(minioClient)) {
|
||||
open class S3FileSystem(provider: S3FileSystemProvider) : BaseFileSystem<S3Path>(provider) {
|
||||
|
||||
private val isOpen = AtomicBoolean(true)
|
||||
|
||||
@@ -20,16 +17,14 @@ class S3FileSystem(
|
||||
return path
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (isOpen.compareAndSet(false, true)) {
|
||||
minioClient.close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun isOpen(): Boolean {
|
||||
return isOpen.get()
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
isOpen.compareAndSet(false, true)
|
||||
}
|
||||
|
||||
override fun getRootDirectories(): Iterable<Path> {
|
||||
return mutableSetOf<Path>(create(separator))
|
||||
}
|
||||
@@ -1,32 +1,21 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import io.minio.*
|
||||
import io.minio.errors.ErrorResponseException
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
import java.net.URI
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.file.*
|
||||
import java.nio.file.attribute.*
|
||||
import java.nio.file.spi.FileSystemProvider
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import kotlin.io.path.absolutePathString
|
||||
import kotlin.io.path.name
|
||||
|
||||
class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemProvider() {
|
||||
abstract class S3FileSystemProvider() : FileSystemProvider() {
|
||||
|
||||
/**
|
||||
* 因为 S3 协议不存在文件夹,所以用户新建的文件夹先保存到内存中
|
||||
*/
|
||||
private val directories = mutableMapOf<String, MutableList<S3Path>>()
|
||||
|
||||
override fun getScheme(): String? {
|
||||
return "s3"
|
||||
}
|
||||
protected val directories = mutableMapOf<String, MutableList<S3Path>>()
|
||||
|
||||
override fun newFileSystem(
|
||||
uri: URI,
|
||||
@@ -49,51 +38,15 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro
|
||||
vararg attrs: FileAttribute<*>
|
||||
): SeekableByteChannel {
|
||||
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
|
||||
if (options.contains(StandardOpenOption.WRITE)) {
|
||||
return S3WriteSeekableByteChannel(Channels.newChannel(createStreamer(path)))
|
||||
return if (options.contains(StandardOpenOption.WRITE)) {
|
||||
S3WriteSeekableByteChannel(getOutputStream(path))
|
||||
} else {
|
||||
val response = minioClient.getObject(
|
||||
GetObjectArgs.builder().bucket(path.bucketName)
|
||||
.`object`(path.objectName).build()
|
||||
)
|
||||
return S3ReadSeekableByteChannel(Channels.newChannel(response), stat(path))
|
||||
S3ReadSeekableByteChannel(getInputStream(path), path.attributes.size())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun createStreamer(path: S3Path): OutputStream {
|
||||
val pis = PipedInputStream()
|
||||
val pos = PipedOutputStream(pis)
|
||||
val exception = AtomicReference<Throwable>()
|
||||
|
||||
val thread = Thread.ofVirtual().start {
|
||||
try {
|
||||
minioClient.putObject(
|
||||
PutObjectArgs.builder()
|
||||
.bucket(path.bucketName)
|
||||
.stream(pis, -1, 32 * 1024 * 1024)
|
||||
.`object`(path.objectName).build()
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
exception.set(e)
|
||||
} finally {
|
||||
IOUtils.closeQuietly(pis)
|
||||
}
|
||||
}
|
||||
|
||||
return object : OutputStream() {
|
||||
override fun write(b: Int) {
|
||||
val exception = exception.get()
|
||||
if (exception != null) throw exception
|
||||
pos.write(b)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
pos.close()
|
||||
if (thread.isAlive) thread.join()
|
||||
}
|
||||
}
|
||||
}
|
||||
abstract fun getOutputStream(path: S3Path): OutputStream
|
||||
abstract fun getInputStream(path: S3Path): InputStream
|
||||
|
||||
override fun newDirectoryStream(
|
||||
dir: Path,
|
||||
@@ -101,7 +54,7 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro
|
||||
): DirectoryStream<Path> {
|
||||
return object : DirectoryStream<Path> {
|
||||
override fun iterator(): MutableIterator<Path> {
|
||||
return files(dir as S3Path).iterator()
|
||||
return fetchChildren(dir as S3Path).iterator()
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
@@ -110,70 +63,8 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro
|
||||
}
|
||||
}
|
||||
|
||||
private fun files(path: S3Path): MutableList<S3Path> {
|
||||
val paths = mutableListOf<S3Path>()
|
||||
abstract fun fetchChildren(path: S3Path): MutableList<S3Path>
|
||||
|
||||
// root
|
||||
if (path.isRoot) {
|
||||
for (bucket in minioClient.listBuckets()) {
|
||||
val p = path.resolve(bucket.name())
|
||||
p.attributes = S3FileAttributes(
|
||||
directory = true,
|
||||
lastModifiedTime = bucket.creationDate().toInstant().toEpochMilli()
|
||||
)
|
||||
paths.add(p)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
var startAfter = StringUtils.EMPTY
|
||||
val maxKeys = 100
|
||||
|
||||
while (true) {
|
||||
val builder = ListObjectsArgs.builder()
|
||||
.bucket(path.bucketName)
|
||||
.maxKeys(maxKeys)
|
||||
.delimiter(path.fileSystem.separator)
|
||||
|
||||
if (path.objectName.isNotBlank()) builder.prefix(path.objectName + path.fileSystem.separator)
|
||||
if (startAfter.isNotBlank()) builder.startAfter(startAfter)
|
||||
|
||||
|
||||
val subPaths = mutableListOf<S3Path>()
|
||||
for (e in minioClient.listObjects(builder.build())) {
|
||||
val item = e.get()
|
||||
val p = path.bucket.resolve(item.objectName())
|
||||
var attributes = p.attributes.copy(
|
||||
directory = item.isDir,
|
||||
regularFile = item.isDir.not(),
|
||||
size = item.size()
|
||||
)
|
||||
if (item.lastModified() != null) {
|
||||
attributes = attributes.copy(lastModifiedTime = item.lastModified().toInstant().toEpochMilli())
|
||||
}
|
||||
p.attributes = attributes
|
||||
|
||||
// 如果是文件夹,那么就要删除内存中的
|
||||
if (attributes.isDirectory) {
|
||||
delete(p)
|
||||
}
|
||||
|
||||
subPaths.add(p)
|
||||
startAfter = item.objectName()
|
||||
}
|
||||
|
||||
paths.addAll(subPaths)
|
||||
|
||||
if (subPaths.size < maxKeys)
|
||||
break
|
||||
|
||||
|
||||
}
|
||||
|
||||
paths.addAll(directories[path.absolutePathString()] ?: emptyList())
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
override fun createDirectory(dir: Path, vararg attrs: FileAttribute<*>) {
|
||||
synchronized(this) {
|
||||
@@ -196,13 +87,20 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro
|
||||
directories[parent.absolutePathString()]?.removeIf { it.name == path.name }
|
||||
}
|
||||
}
|
||||
return
|
||||
delete(path, true)
|
||||
} else {
|
||||
delete(path, false)
|
||||
}
|
||||
minioClient.removeObject(
|
||||
RemoveObjectArgs.builder().bucket(path.bucketName).`object`(path.objectName).build()
|
||||
)
|
||||
}
|
||||
|
||||
override fun checkAccess(path: Path, vararg modes: AccessMode) {
|
||||
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
|
||||
checkAccess(path, *modes)
|
||||
}
|
||||
|
||||
abstract fun delete(path: S3Path, isDirectory: Boolean)
|
||||
abstract fun checkAccess(path: S3Path, vararg modes: AccessMode)
|
||||
|
||||
override fun copy(
|
||||
source: Path?,
|
||||
target: Path?,
|
||||
@@ -231,25 +129,6 @@ class S3FileSystemProvider(private val minioClient: MinioClient) : FileSystemPro
|
||||
throw UnsupportedOperationException()
|
||||
}
|
||||
|
||||
override fun checkAccess(path: Path, vararg modes: AccessMode) {
|
||||
if (path !is S3Path) throw UnsupportedOperationException("path must be a S3Path")
|
||||
|
||||
try {
|
||||
stat(path)
|
||||
} catch (e: ErrorResponseException) {
|
||||
throw NoSuchFileException(e.errorResponse().message())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun stat(path: S3Path): StatObjectResponse {
|
||||
return minioClient.statObject(
|
||||
StatObjectArgs.builder()
|
||||
.`object`(path.objectName)
|
||||
.bucket(path.bucketName).build()
|
||||
)
|
||||
}
|
||||
|
||||
override fun <V : FileAttributeView> getFileAttributeView(
|
||||
path: Path,
|
||||
type: Class<V>,
|
||||
@@ -1,4 +1,4 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import app.termora.transfer.WithPathAttributes
|
||||
import org.apache.sshd.common.file.util.BasePath
|
||||
@@ -6,7 +6,7 @@ import java.nio.file.LinkOption
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.absolutePathString
|
||||
|
||||
class S3Path(
|
||||
open class S3Path(
|
||||
fileSystem: S3FileSystem,
|
||||
root: String?,
|
||||
names: List<String>,
|
||||
@@ -20,27 +20,27 @@ class S3Path(
|
||||
/**
|
||||
* 是否是 Bucket
|
||||
*/
|
||||
val isBucket get() = parent != null && parent?.parent == null
|
||||
open val isBucket get() = parent != null && parent?.parent == null
|
||||
|
||||
/**
|
||||
* 是否是根
|
||||
*/
|
||||
val isRoot get() = absolutePathString() == separator
|
||||
open val isRoot get() = absolutePathString() == separator
|
||||
|
||||
/**
|
||||
* Bucket Name
|
||||
*/
|
||||
val bucketName: String get() = names.first()
|
||||
open val bucketName: String get() = names.first()
|
||||
|
||||
/**
|
||||
* 获取 Bucket
|
||||
*/
|
||||
val bucket: S3Path get() = fileSystem.getPath(root, bucketName)
|
||||
open val bucket: S3Path get() = fileSystem.getPath(root, bucketName)
|
||||
|
||||
/**
|
||||
* 获取所在 Bucket 的路径
|
||||
*/
|
||||
val objectName: String get() = names.subList(1, names.size).joinToString(separator)
|
||||
open val objectName: String get() = names.subList(1, names.size).joinToString(separator)
|
||||
|
||||
override fun getCustomType(): String? {
|
||||
if (isBucket) return "Bucket"
|
||||
@@ -1,16 +1,13 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import io.minio.StatObjectResponse
|
||||
import org.apache.commons.io.IOUtils
|
||||
import java.io.InputStream
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.ReadableByteChannel
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
|
||||
class S3ReadSeekableByteChannel(
|
||||
private val channel: ReadableByteChannel,
|
||||
private val stat: StatObjectResponse
|
||||
) : SeekableByteChannel {
|
||||
|
||||
open class S3ReadSeekableByteChannel(input: InputStream, private val size: Long) : SeekableByteChannel {
|
||||
private val channel = Channels.newChannel(input)
|
||||
private var position: Long = 0
|
||||
|
||||
override fun read(dst: ByteBuffer): Int {
|
||||
@@ -34,7 +31,7 @@ class S3ReadSeekableByteChannel(
|
||||
}
|
||||
|
||||
override fun size(): Long {
|
||||
return stat.size()
|
||||
return size
|
||||
}
|
||||
|
||||
override fun truncate(size: Long): SeekableByteChannel {
|
||||
@@ -1,14 +1,13 @@
|
||||
package app.termora.plugins.s3
|
||||
package app.termora.transfer.s3
|
||||
|
||||
import org.apache.commons.io.IOUtils
|
||||
import java.io.OutputStream
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.Channels
|
||||
import java.nio.channels.SeekableByteChannel
|
||||
import java.nio.channels.WritableByteChannel
|
||||
|
||||
class S3WriteSeekableByteChannel(
|
||||
private val channel: WritableByteChannel,
|
||||
) : SeekableByteChannel {
|
||||
|
||||
open class S3WriteSeekableByteChannel(output: OutputStream) : SeekableByteChannel {
|
||||
private val channel = Channels.newChannel(output)
|
||||
|
||||
override fun read(dst: ByteBuffer): Int {
|
||||
throw UnsupportedOperationException("read not supported")
|
||||
@@ -1,39 +0,0 @@
|
||||
package app.termora
|
||||
|
||||
import app.termora.database.DataEntity
|
||||
import app.termora.database.DataType
|
||||
import app.termora.database.OwnerType
|
||||
import app.termora.database.SettingEntity
|
||||
import org.jetbrains.exposed.v1.jdbc.Database
|
||||
import org.jetbrains.exposed.v1.jdbc.SchemaUtils
|
||||
import org.jetbrains.exposed.v1.jdbc.insert
|
||||
import org.jetbrains.exposed.v1.jdbc.transactions.transaction
|
||||
import kotlin.test.Test
|
||||
|
||||
|
||||
class ExposedTest {
|
||||
|
||||
|
||||
@Test
|
||||
fun test() {
|
||||
val database = Database.connect("jdbc:h2:mem:test", driver = "org.h2.Driver", user = "sa")
|
||||
|
||||
transaction(database) {
|
||||
SchemaUtils.create(DataEntity, SettingEntity)
|
||||
|
||||
println(DataEntity.insert {
|
||||
it[ownerId] = "Test"
|
||||
it[ownerType] = OwnerType.User.name
|
||||
it[type] = DataType.KeywordHighlight.name
|
||||
it[data] = "hello 中文".repeat(10000)
|
||||
} get DataEntity.id)
|
||||
|
||||
println(SettingEntity.insert {
|
||||
it[name] = "Test"
|
||||
it[value] = "hello 中文".repeat(10000)
|
||||
} get SettingEntity.id)
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ class HostTest {
|
||||
"""
|
||||
{
|
||||
"name": "test",
|
||||
"protocol": SSHProtocolProvider.PROTOCOL,
|
||||
"protocol": "SSH",
|
||||
"test": ""
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
package app.termora.account
|
||||
|
||||
import kotlin.test.Test
|
||||
|
||||
class ServerManagerTest {
|
||||
@Test
|
||||
fun test() {
|
||||
ServerManager.getInstance().login(
|
||||
Server(
|
||||
name = "test",
|
||||
server = "http://127.0.0.1:8080"
|
||||
), "admin", "admin"
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package app.termora.vfs2.sftp
|
||||
|
||||
import app.termora.SSHDTest
|
||||
import app.termora.randomUUID
|
||||
import org.apache.commons.vfs2.*
|
||||
import org.apache.commons.vfs2.impl.DefaultFileSystemManager
|
||||
import org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory
|
||||
import java.io.File
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class MySftpFileProviderTest : SSHDTest() {
|
||||
|
||||
companion object {
|
||||
init {
|
||||
val fileSystemManager = DefaultFileSystemManager()
|
||||
fileSystemManager.addProvider("sftp", MySftpFileProvider.instance)
|
||||
fileSystemManager.addProvider("file", DefaultLocalFileProvider())
|
||||
fileSystemManager.init()
|
||||
VFS.setManager(fileSystemManager)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testSetExecutable() {
|
||||
val file = newFileObject("/config/test.txt")
|
||||
file.createFile()
|
||||
file.refresh()
|
||||
assertFalse(file.isExecutable)
|
||||
file.setExecutable(true, false)
|
||||
file.refresh()
|
||||
assertTrue(file.isExecutable)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testCreateFile() {
|
||||
val file = newFileObject("/config/test.txt")
|
||||
assertFalse(file.exists())
|
||||
file.createFile()
|
||||
assertTrue(file.exists())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testWriteAndReadFile() {
|
||||
val file = newFileObject("/config/test.txt")
|
||||
file.createFile()
|
||||
assertFalse(file.content.isOpen)
|
||||
|
||||
val os = file.content.outputStream
|
||||
os.write("test".toByteArray())
|
||||
os.flush()
|
||||
assertTrue(file.content.isOpen)
|
||||
|
||||
os.close()
|
||||
assertFalse(file.content.isOpen)
|
||||
|
||||
val input = file.content.inputStream
|
||||
assertEquals("test", String(input.readAllBytes()))
|
||||
assertTrue(file.content.isOpen)
|
||||
input.close()
|
||||
assertFalse(file.content.isOpen)
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testCreateFolder() {
|
||||
val file = newFileObject("/config/test")
|
||||
assertFalse(file.exists())
|
||||
file.createFolder()
|
||||
assertTrue(file.exists())
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
fun testSftpClient() {
|
||||
val session = newClientSession()
|
||||
val client = SftpClientFactory.instance().createSftpClient(session)
|
||||
assertTrue(client.isOpen)
|
||||
session.close()
|
||||
assertFalse(client.isOpen)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testCopy() {
|
||||
val file = newFileObject("/config/sshd.pid")
|
||||
val filepath = File("build", randomUUID())
|
||||
val localFile = getVFS().resolveFile("file://${filepath.absolutePath}")
|
||||
|
||||
localFile.copyFrom(file, Selectors.SELECT_ALL)
|
||||
assertEquals(
|
||||
file.content.getString(Charsets.UTF_8),
|
||||
localFile.content.getString(Charsets.UTF_8)
|
||||
)
|
||||
|
||||
localFile.delete()
|
||||
}
|
||||
|
||||
private fun getVFS(): FileSystemManager {
|
||||
return VFS.getManager()
|
||||
}
|
||||
|
||||
private fun newFileObject(path: String): FileObject {
|
||||
val vfs = getVFS()
|
||||
val fileSystemOptions = FileSystemOptions()
|
||||
MySftpFileSystemConfigBuilder.getInstance()
|
||||
.setSftpFileSystem(fileSystemOptions, SftpClientFactory.instance().createSftpFileSystem(newClientSession()))
|
||||
return vfs.resolveFile("sftp://${path}", fileSystemOptions)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user