native importer module wip

app-android/tutashared/src/main/java/de/tutao/tutashared/generated_ipc/NativeCryptoFacadeReceiveDispatcher.kt

@@ -2,17 +2,16 @@
 
 
 @file:Suppress("NAME_SHADOWING")
-
 package de.tutao.tutashared.ipc
 
-import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.Json
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
 
 class NativeCryptoFacadeReceiveDispatcher(
 	private val json: Json,
 	private val facade: NativeCryptoFacade,
 ) {
-
+	
 	suspend fun dispatch(method: String, arg: List<String>): String {
 		when (method) {
 			"rsaEncrypt" -> {
@@ -26,7 +25,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"rsaDecrypt" -> {
 				val privateKey: RsaPrivateKey = json.decodeFromString(arg[0])
 				val data: DataWrapper = json.decodeFromString(arg[1])
@@ -36,7 +34,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"aesEncryptFile" -> {
 				val key: DataWrapper = json.decodeFromString(arg[0])
 				val fileUri: String = json.decodeFromString(arg[1])
@@ -48,7 +45,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"aesDecryptFile" -> {
 				val key: DataWrapper = json.decodeFromString(arg[0])
 				val fileUri: String = json.decodeFromString(arg[1])
@@ -58,7 +54,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"argon2idGeneratePassphraseKey" -> {
 				val passphrase: String = json.decodeFromString(arg[0])
 				val salt: DataWrapper = json.decodeFromString(arg[1])
@@ -68,15 +63,13 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"generateKyberKeypair" -> {
 				val seed: DataWrapper = json.decodeFromString(arg[0])
 				val result: KyberKeyPair = this.facade.generateKyberKeypair(
 					seed,
 				)
 				return json.encodeToString(result)
 			}
-
 			"kyberEncapsulate" -> {
 				val publicKey: KyberPublicKey = json.decodeFromString(arg[0])
 				val seed: DataWrapper = json.decodeFromString(arg[1])
@@ -86,7 +79,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			"kyberDecapsulate" -> {
 				val privateKey: KyberPrivateKey = json.decodeFromString(arg[0])
 				val ciphertext: DataWrapper = json.decodeFromString(arg[1])
@@ -96,7 +88,6 @@ class NativeCryptoFacadeReceiveDispatcher(
 				)
 				return json.encodeToString(result)
 			}
-
 			else -> throw Error("unknown method for NativeCryptoFacade: $method")
 		}
 	}

app-android/tutashared/src/main/java/de/tutao/tutashared/generated_ipc/PersistedCredentials.kt

@@ -3,7 +3,8 @@
 
 package de.tutao.tutashared.ipc
 
-import kotlinx.serialization.Serializable
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
 
 
 /**

app-android/tutashared/src/main/java/de/tutao/tutashared/generated_ipc/UnencryptedCredentials.kt

@@ -3,7 +3,8 @@
 
 package de.tutao.tutashared.ipc
 
-import kotlinx.serialization.Serializable
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
 
 
 /**

buildSrc/DesktopBuilder.js

@@ -121,7 +121,10 @@ async function rollupDesktop(dirname, outDir, version, platform, architecture, d
 		input: [path.join(dirname, "src/common/desktop/DesktopMain.ts"), path.join(dirname, "src/common/desktop/sqlworker.ts")],
 		// some transitive dep of a transitive dev-dep requires https://www.npmjs.com/package/url
 		// which rollup for some reason won't distinguish from the node builtin.
-		external: ["url", "util", "path", "fs", "os", "http", "https", "crypto", "child_process", "electron"],
+		external: (id, parent, isResolved) => {
+			if (parent != null && parent.endsWith("node-mimimi/dist/binding.cjs")) return true
+			return ["url", "util", "path", "fs", "os", "http", "https", "crypto", "child_process", "electron"].includes(id)
+		},
 		preserveEntrySignatures: false,
 		plugins: [
 			copyNativeModulePlugin({
@@ -131,6 +134,16 @@ async function rollupDesktop(dirname, outDir, version, platform, architecture, d
 				architecture,
 				nodeModule: "better-sqlite3",
 			}),
+			{
+				// todo: this needs to work everywhere
+				name: "copy-mimimi-plugin",
+				async buildStart() {
+					const normalDst = path.join(path.normalize("./build/desktop/"), "node-mimimi.linux-x64-gnu.node")
+					const dstDir = path.dirname(normalDst)
+					await fs.promises.mkdir(dstDir, { recursive: true })
+					await fs.promises.copyFile("./packages/node-mimimi/dist/node-mimimi.linux-x64-gnu.node", normalDst)
+				},
+			},
 			typescript({
 				tsconfig: "tsconfig.json",
 				outDir,

buildSrc/DevBuild.js

@@ -4,7 +4,7 @@ import { build as esbuild } from "esbuild"
 import { getTutanotaAppVersion, runStep, writeFile } from "./buildUtils.js"
 import "zx/globals"
 import * as env from "./env.js"
-import { externalTranslationsPlugin, libDeps, preludeEnvPlugin, sqliteNativePlugin } from "./esbuildUtils.js"
+import { externalTranslationsPlugin, libDeps, mimimiNativePlugin, preludeEnvPlugin, sqliteNativePlugin } from "./esbuildUtils.js"
 import { fileURLToPath } from "node:url"
 import * as LaunchHtml from "./LaunchHtml.js"
 import os from "node:os"
@@ -173,9 +173,9 @@ async function buildDesktopPart({ version, app }) {
 			format: "cjs",
 			sourcemap: "linked",
 			platform: "node",
-			external: ["electron"],
+			external: ["electron", "*.node"],
 			banner: {
-				js: `globalThis.buildOptions = globalThis.buildOptions ?? {}
+				js: `globalThis.buildOptions = globalThis.buildOptions ?? {} 
 globalThis.buildOptions.sqliteNativePath = "./better-sqlite3.node";`,
 			},
 			plugins: [
@@ -187,6 +187,10 @@ globalThis.buildOptions.sqliteNativePath = "./better-sqlite3.node";`,
 					architecture: process.arch,
 					nativeBindingPath: "./better_sqlite3.node",
 				}),
+				mimimiNativePlugin({
+					dstPath: `./${buildDir}/desktop/`,
+					platform: process.platform,
+				}),
 				preludeEnvPlugin(env.create({ staticUrl: null, version, mode: "Desktop", dist: false, domainConfigs })),
 				externalTranslationsPlugin(),
 			],

buildSrc/RustGenerator.js

@@ -13,7 +13,7 @@ import { AssociationType, Type } from "../src/common/api/common/EntityConstants.
  */
 export function generateRustType({ type, modelName }) {
 	let typeName = mapTypeName(type.name, modelName)
-	let buf = `#[derive(uniffi::Record, Clone, Serialize, Deserialize, Debug)]
+	let buf = `#[derive(uniffi::Record, Clone, Serialize, Deserialize, Debug, PartialEq)]
 pub struct ${typeName} {\n`
 	for (let [valueName, valueProperties] of Object.entries(type.values)) {
 		const rustType = rustValueType(valueName, type, valueProperties)
@@ -59,20 +59,81 @@ pub struct ${typeName} {\n`
 	if (type.encrypted || Object.values(type.values).some((v) => v.encrypted)) {
 		buf += `\tpub _finalIvs: HashMap<String, FinalIv>,\n`
 	}
+
 	buf += "}"
+	buf += `
+impl Entity for ${typeName} {
+	fn type_ref() -> TypeRef {
+		TypeRef {
+			app: "${modelName}",
+			type_: "${typeName}",
+		}
+	}
+}
+`
+
+	return buf + "\n\n"
+}
 
-	buf += "\n\n"
+export function generateRustServiceDefinition(appName, appVersion, services) {
+	let imports = new Set([
+		"#![allow(unused_imports, dead_code, unused_variables)]",
+		"use crate::ApiCallError;",
+		"use crate::entities::Entity;",
+		"use crate::services::{PostService, GetService, PutService, DeleteService, Service, Executor, ExtraServiceParams};",
+		"use crate::rest_client::HttpMethod;",
+		"use crate::services::hidden::Nothing;",
+	])
+	const code = services
+		.map((s) => {
+			let serviceDefinition = `
+pub struct ${s.name};
 
-	buf += `impl Entity for ${typeName} {\n`
-	buf += "\tfn type_ref() -> TypeRef {\n"
-	buf += `\t\tTypeRef {\n`
-	buf += `\t\t\tapp: "${modelName}",\n`
-	buf += `\t\t\ttype_: "${typeName}",\n`
-	buf += `\t\t}\n`
-	buf += "\t}\n"
-	buf += "}"
+crate::service_impl!(base, ${s.name}, "${appName}/${s.name.toLowerCase()}", ${appVersion});
+`
+
+			function getTypeRef(dataType) {
+				if (dataType) {
+					return `Some(${dataType}::type_ref())`
+				} else {
+					return "None"
+				}
+			}
+
+			function addImports(appName, input, output) {
+				if (input) {
+					imports.add(`use crate::entities::${appName}::${input};`)
+				}
+				if (output) {
+					imports.add(`use crate::entities::${appName}::${output};`)
+				}
+			}
+
+			function makeImpl(name, input, output) {
+				addImports(appName, input, output)
+				return `crate::service_impl!(${name}, ${s.name}, ${input ?? "()"}, ${output ?? "()"});\n`
+			}
+
+			if (s.bodyTypes.POST_IN || s.bodyTypes.POST_OUT) {
+				serviceDefinition += makeImpl("POST", s.bodyTypes.POST_IN, s.bodyTypes.POST_OUT)
+			}
+
+			if (s.bodyTypes.GET_IN || s.bodyTypes.GET_OUT) {
+				serviceDefinition += makeImpl("GET", s.bodyTypes.GET_IN, s.bodyTypes.GET_OUT)
+			}
+
+			if (s.bodyTypes.PUT_IN || s.bodyTypes.PUT_OUT) {
+				serviceDefinition += makeImpl("PUT", s.bodyTypes.PUT_IN, s.bodyTypes.PUT_OUT)
+			}
+
+			if (s.bodyTypes.DELETE_IN || s.bodyTypes.DELETE_OUT) {
+				serviceDefinition += makeImpl("DELETE", s.bodyTypes.DELETE_IN, s.bodyTypes.DELETE_OUt)
+			}
 
-	return buf
+			return serviceDefinition
+		})
+		.join("\n")
+	return Array.from(imports).join("\n") + code
 }
 
 /**

buildSrc/esbuildUtils.js

@@ -6,7 +6,7 @@ import { aliasPath as esbuildPluginAliasPath } from "esbuild-plugin-alias-path"
 
 /**
  * Little plugin that obtains compiled better-sqlite3, copies it to dstPath and sets the path to nativeBindingPath.
- * We do not use default file loader from esbuild, it is much simpler and reliable to do it manually and it doesn't work for dynamic import (like in this case)
+ * We do not use default file loader from esbuild, it is much simpler and reliable to do it manually, and it doesn't work for dynamic import (like in this case)
  * anyway.
  * It will also replace `buildOptions.sqliteNativePath` with the nativeBindingPath
  */
@@ -37,6 +37,41 @@ export function sqliteNativePlugin({ environment, dstPath, nativeBindingPath, pl
 	}
 }
 
+export function mimimiNativePlugin({ dstPath, platform }) {
+	return {
+		name: "mimimi-native-plugin",
+		setup(build) {
+			const options = build.initialOptions
+			options.define = options.define ?? {}
+
+			build.onStart(async () => {
+				let nativeBinaryName
+				switch (platform) {
+					case "linux":
+						nativeBinaryName = "node-mimimi.linux-x64-gnu.node"
+						break
+					case "win32":
+						nativeBinaryName = "node-mimimi.win32-x64-msvc.node"
+						break
+					case "darwin":
+						nativeBinaryName = "node-mimimi.darwin-universal.node"
+						break
+					default:
+						throw Error(`could not find node-mimimi binary: platform ${platform} is unknown`)
+				}
+
+				// Replace mentions of buildOptions.mimimiNativePath with the actual path
+				options.define["buildOptions.mimimiNativePath"] = `"./${nativeBinaryName}"`
+
+				const nativeBinarySourcePath = path.join(process.cwd(), "./packages/node-mimimi/dist", nativeBinaryName)
+
+				await fs.promises.mkdir(path.dirname(dstPath), { recursive: true })
+				await fs.promises.copyFile(nativeBinarySourcePath, path.join(process.cwd(), dstPath, nativeBinaryName))
+			})
+		},
+	}
+}
+
 /** Little plugin that replaces imports for libs from dependencyMap with their prebuilt versions in libs directory. */
 export function libDeps(prefix = ".") {
 	const absoluteDependencyMap = Object.fromEntries(

buildSrc/packageBuilderFunctions.js

@@ -12,13 +12,13 @@ export async function buildRuntimePackages() {
 	// tsconfig is rather JSON5, if it becomes a problem switch to JSON5 parser here
 	const tsconfig = JSON.parse(await fs.readFile("tsconfig.json", { encoding: "utf-8" }))
 	const packagePaths = tsconfig.references.map((ref) => ref.path)
-	await $`npx tsc -b ${packagePaths}`
+	await Promise.all(packagePaths.map((dir) => $`cd ${dir} && npm run build`))
 }
 
 /**
  * Build all packages in packages directory.
  */
 export async function buildPackages(pathPrefix = ".") {
 	const packages = await glob(`${pathPrefix}/packages/*`, { deep: 1, onlyDirectories: true })
-	await $`npx tsc -b ${packages}`
+	await Promise.all(packages.map((dir) => $`cd ${dir} && npm run build`))
 }

ipc-schema/facades/ImapImportSystemFacade.json

@@ -0,0 +1,28 @@
+{
+	"name": "ImapImportSystemFacade",
+	"type": "facade",
+	"senders": ["web"],
+	"receivers": ["desktop"],
+	"doc": "Facade implemented by the native desktop client starting and stopping an IMAP import.",
+	"methods": {
+		"setup": {
+			"doc": "Initializing the IMAP import.",
+			"arg": [
+				{
+					"imapCredentials": "ImapCredentials"
+				}
+			],
+			"ret": "void"
+		},
+		"startImport": {
+			"doc": "Start the IMAP import.",
+			"arg": [],
+			"ret": "void"
+		},
+		"stopImport": {
+			"doc": "Stop a running IMAP import.",
+			"arg": [],
+			"ret": "void"
+		}
+	}
+}

ipc-schema/types/ImapCredentials.json

@@ -0,0 +1,7 @@
+{
+	"name": "ImapCredentials",
+	"type": "typeref",
+	"location": {
+		"typescript": "../packages/node-mimimi/dist/binding.js"
+	}
+}

ipc-schema/types/MailBundle.json

@@ -2,6 +2,6 @@
 	"name": "MailBundle",
 	"type": "typeref",
 	"location": {
-		"typescript": "../src/mail-app/mail/export/Bundler.js"
+		"typescript": "../src/common/mailFunctionality/SharedMailUtils.js"
 	}
 }