From b3955791820af43edcfa34ef3967da718d9c61b5 Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Mon, 24 Jul 2023 18:12:43 +0200 Subject: [PATCH 1/6] [MCPODS-5992] new test kotlin notebook --- docs/TESTNOTEBOOK.ipynb | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/TESTNOTEBOOK.ipynb diff --git a/docs/TESTNOTEBOOK.ipynb b/docs/TESTNOTEBOOK.ipynb new file mode 100644 index 000000000..e2dd09f9b --- /dev/null +++ b/docs/TESTNOTEBOOK.ipynb @@ -0,0 +1,78 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "This is the initial draft of a Kotlin Notebook, basically the same as Jupyter notebooks, where code snippets can also be included in the markdown docs, and executed. This is more popular for workshops with smaller units of code needing execution, but theoretically there is a setting that allows new notebooks to import project dependencies, so doing something like ramping up Naksha locally should be achievable in a code block in notebooks. Furthermore the ability to render other formats like HTML webpage or LaTeX mathematical formulas directly inside one notebook file might be a good presentation for a product.\n", + "\n", + "Follow the guide at https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/ to install the plugin for IntelliJ." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [ + { + "data": { + "text/plain": "2" + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1+1" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-07-24T16:11:16.215767348Z", + "start_time": "2023-07-24T16:11:16.107824611Z" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Currently Jetbrains has [this notebook](https://datalore.jetbrains.com/report/static/sBgNeHxfaKG1tVRDWl3KqU/DMtWM05XWOT2Oi6yxZjVkJ) to showcase the capabilities of these Kotlin notebooks." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Kotlin", + "language": "kotlin", + "name": "kotlin" + }, + "language_info": { + "name": "kotlin", + "version": "1.8.20", + "mimetype": "text/x-kotlin", + "file_extension": ".kt", + "pygments_lexer": "kotlin", + "codemirror_mode": "text/x-kotlin", + "nbconvert_exporter": "" + }, + "ktnbPluginMetadata": { + "isAddProjectLibrariesToClasspath": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From c1fab845820aaf7b31ae923bc5aa1c7a728b4b69 Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Fri, 28 Jul 2023 14:14:37 +0200 Subject: [PATCH 2/6] [MCPODS-5992] possibility to change the password in test and in notebook, notebook not working yet --- docs/TESTNOTEBOOK.ipynb | 110 +++++++++++++++--- .../naksha/app/service/NakshaHubTest.java | 8 +- 2 files changed, 99 insertions(+), 19 deletions(-) diff --git a/docs/TESTNOTEBOOK.ipynb b/docs/TESTNOTEBOOK.ipynb index e2dd09f9b..63f36563a 100644 --- a/docs/TESTNOTEBOOK.ipynb +++ b/docs/TESTNOTEBOOK.ipynb @@ -5,45 +5,121 @@ "source": [ "This is the initial draft of a Kotlin Notebook, basically the same as Jupyter notebooks, where code snippets can also be included in the markdown docs, and executed. This is more popular for workshops with smaller units of code needing execution, but theoretically there is a setting that allows new notebooks to import project dependencies, so doing something like ramping up Naksha locally should be achievable in a code block in notebooks. Furthermore the ability to render other formats like HTML webpage or LaTeX mathematical formulas directly inside one notebook file might be a good presentation for a product.\n", "\n", - "Follow the guide at https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/ to install the plugin for IntelliJ." + "Follow the guide at https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/ to install the plugin for IntelliJ.\n", + "\n", + "Currently Jetbrains has [this notebook](https://datalore.jetbrains.com/report/static/sBgNeHxfaKG1tVRDWl3KqU/DMtWM05XWOT2Oi6yxZjVkJ) to showcase the capabilities of these Kotlin notebooks." ], "metadata": { "collapsed": false } }, { - "cell_type": "code", - "execution_count": 4, - "outputs": [ - { - "data": { - "text/plain": "2" - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } + "cell_type": "markdown", + "source": [ + "To setup a Naksha Hub server on your local machine, first define the main Postgres password:" ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [], "source": [ - "1+1" + "var password = \"postgres\"" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-07-24T16:11:16.215767348Z", - "start_time": "2023-07-24T16:11:16.107824611Z" + "end_time": "2023-07-28T12:10:48.605019065Z", + "start_time": "2023-07-28T12:10:48.493613134Z" } } }, { "cell_type": "markdown", "source": [ - "Currently Jetbrains has [this notebook](https://datalore.jetbrains.com/report/static/sBgNeHxfaKG1tVRDWl3KqU/DMtWM05XWOT2Oi6yxZjVkJ) to showcase the capabilities of these Kotlin notebooks." + "Then run the code block below to start the Hub:" ], "metadata": { "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n", + "com.here.naksha.lib.core.exceptions.UncheckedException: java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n", + "\tat com.here.naksha.lib.core.exceptions.UncheckedException.unchecked(UncheckedException.java:43)\n", + "\tat com.here.naksha.lib.psql.PsqlStorage.init(PsqlStorage.java:195)\n", + "\tat com.here.naksha.app.service.NakshaHub.(NakshaHub.java:168)\n", + "\tat com.here.naksha.app.service.NakshaHub.newHub(NakshaHub.java:130)\n", + "\tat Line_3_jupyter.(Line_3.jupyter.kts:4)\n", + "\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n", + "\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)\n", + "\tat java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n", + "\tat java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)\n", + "\tat java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)\n", + "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.evalWithConfigAndOtherScriptsResults(BasicJvmScriptEvaluator.kt:105)\n", + "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke$suspendImpl(BasicJvmScriptEvaluator.kt:47)\n", + "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke(BasicJvmScriptEvaluator.kt)\n", + "\tat kotlin.script.experimental.jvm.BasicJvmReplEvaluator.eval(BasicJvmReplEvaluator.kt:49)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl$eval$resultWithDiagnostics$1.invokeSuspend(InternalEvaluatorImpl.kt:103)\n", + "\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)\n", + "\tat kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:106)\n", + "\tat kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:284)\n", + "\tat kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:85)\n", + "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:59)\n", + "\tat kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)\n", + "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:38)\n", + "\tat kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl.eval(InternalEvaluatorImpl.kt:103)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:75)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:73)\n", + "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.withHost(repl.kt:666)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl.execute(CellExecutorImpl.kt:73)\n", + "\tat org.jetbrains.kotlinx.jupyter.repl.CellExecutor$DefaultImpls.execute$default(CellExecutor.kt:15)\n", + "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl$evalEx$1.invoke(repl.kt:479)\n", + "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl$evalEx$1.invoke(repl.kt:470)\n", + "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.withEvalContext(repl.kt:433)\n", + "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.evalEx(repl.kt:470)\n", + "\tat org.jetbrains.kotlinx.jupyter.messaging.ProtocolKt$shellMessagesHandler$2$res$1.invoke(protocol.kt:318)\n", + "\tat org.jetbrains.kotlinx.jupyter.messaging.ProtocolKt$shellMessagesHandler$2$res$1.invoke(protocol.kt:312)\n", + "\tat org.jetbrains.kotlinx.jupyter.JupyterExecutorImpl$runExecution$execThread$1.invoke(execution.kt:37)\n", + "\tat org.jetbrains.kotlinx.jupyter.JupyterExecutorImpl$runExecution$execThread$1.invoke(execution.kt:32)\n", + "\tat kotlin.concurrent.ThreadsKt$thread$thread$1.run(Thread.kt:30)\n", + "Caused by: java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n", + "\tat com.here.naksha.lib.core.NakshaLogger.atInfo(NakshaLogger.java:415)\n", + "\tat com.here.naksha.lib.psql.PsqlStorage.init(PsqlStorage.java:173)\n", + "\t... 36 more\n" + ] + } + ], + "source": [ + "import com.here.naksha.app.service.NakshaHub\n", + "import java.util.concurrent.TimeUnit\n", + "\n", + "var hub: NakshaHub? = NakshaHub.newHub(\n", + " \"jdbc:postgresql://localhost/postgres?user=postgres&password=$password\",\n", + " \"local\"\n", + ")\n", + "hub!!.start()\n", + "hub!!.join(TimeUnit.SECONDS.toMillis(60))\n" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-07-28T12:12:46.202817846Z", + "start_time": "2023-07-28T12:12:45.881557242Z" + } + } + }, { "cell_type": "code", "execution_count": null, @@ -70,7 +146,7 @@ "nbconvert_exporter": "" }, "ktnbPluginMetadata": { - "isAddProjectLibrariesToClasspath": false + "isBuildProject": true } }, "nbformat": 4, diff --git a/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java b/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java index 45151f763..b69a363e2 100644 --- a/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java +++ b/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java @@ -28,7 +28,11 @@ class NakshaHubTest { @BeforeAll static void prepare() { - hub = newHub("jdbc:postgresql://localhost/postgres?user=postgres&password=password", "local"); + String password = System.getenv("TEST_NAKSHA_PSQL_PASS"); + if (password == null) password = "password"; + hub = newHub( + "jdbc:postgresql://localhost/postgres?user=postgres&password=" + password, + "local"); // this string concat only happens once, its ok ;) hub.start(); } @@ -44,6 +48,6 @@ void startup() throws InterruptedException { static void close() throws InterruptedException { // TODO: Find a way to gracefully shutdown the server. // To do some manual testing with the running service, uncomment this: - //hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(60)); + hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(60)); } } From fcfb8e750324835f81717f5bb97ed374b53801d6 Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Fri, 28 Jul 2023 18:45:56 +0200 Subject: [PATCH 3/6] halfway though naksha storage maintain --- .../com/here/naksha/lib/psql/PsqlStorage.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java index 9687b0164..cad4bf7f2 100644 --- a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java +++ b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java @@ -22,6 +22,7 @@ import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static com.here.naksha.lib.psql.SQL.escapeId; +import com.here.naksha.lib.core.NakshaAdminCollection; import com.here.naksha.lib.core.NakshaVersion; import com.here.naksha.lib.core.lambdas.Pe1; import com.here.naksha.lib.core.models.TxSignalSet; @@ -204,7 +205,34 @@ public void init() { */ @Override public void maintain() { - // throw new UnsupportedOperationException("maintain"); + NakshaAdminCollection.STORAGES.getId(); + try (final Connection conn = dataSource.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.execute(new StringBuilder() + .append("SET SESSION search_path TO \"") + .append(getSchema()) + .append("\",\"public\",\"topology\";") + .toString()); + stmt.execute(createPartitionOfOneDay(0)); + stmt.execute(createPartitionOfOneDay(1)); + stmt.execute(createPartitionOfOneDay(2)); + //TODO delete old partitions + } + } catch (Throwable t) { + throw unchecked(t); + } + } + + private String createPartitionOfOneDay(int dayplus) { + return new StringBuilder() + .append("SELECT ") + .append(getSchema()) + .append(".__naksha_create_hst_partition_for_day('") + .append(NakshaAdminCollection.STORAGES.getId()) + .append("',current_timestamp+'") + .append(dayplus) + .append(" days');") + .toString(); } /** From 3434e206b31717bee8f7f77a3221207c9dee8306 Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Wed, 2 Aug 2023 15:37:06 +0200 Subject: [PATCH 4/6] [MCPODS-6074] delete partition of 30 day old --- .../naksha/app/service/NakshaHubTest.java | 2 +- .../com/here/naksha/lib/psql/PsqlStorage.java | 19 +++++++++++++++---- .../src/main/resources/naksha_ext.sql | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java b/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java index b69a363e2..6d67bceff 100644 --- a/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java +++ b/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java @@ -48,6 +48,6 @@ void startup() throws InterruptedException { static void close() throws InterruptedException { // TODO: Find a way to gracefully shutdown the server. // To do some manual testing with the running service, uncomment this: - hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(60)); + //hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(10)); } } diff --git a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java index eb0d454e4..8eb73dba6 100644 --- a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java +++ b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java @@ -228,7 +228,6 @@ public void init() { */ @Override public void maintain() { - NakshaAdminCollection.STORAGES.getId(); try (final Connection conn = dataSource.getConnection()) { try (final Statement stmt = conn.createStatement()) { stmt.execute(new StringBuilder() @@ -239,21 +238,33 @@ public void maintain() { stmt.execute(createPartitionOfOneDay(0)); stmt.execute(createPartitionOfOneDay(1)); stmt.execute(createPartitionOfOneDay(2)); - //TODO delete old partitions + stmt.execute(deletePartitionOfOneDay(30)); } } catch (Throwable t) { throw unchecked(t); } } - private String createPartitionOfOneDay(int dayplus) { + private String createPartitionOfOneDay(int dayPlus) { return new StringBuilder() .append("SELECT ") .append(getSchema()) .append(".__naksha_create_hst_partition_for_day('") .append(NakshaAdminCollection.STORAGES.getId()) .append("',current_timestamp+'") - .append(dayplus) + .append(dayPlus) + .append(" days');") + .toString(); + } + + private String deletePartitionOfOneDay(int dayOld) { + return new StringBuilder() + .append("SELECT ") + .append(getSchema()) + .append(".__naksha_delete_hst_partition_for_day('") + .append(NakshaAdminCollection.STORAGES.getId()) + .append("',current_timestamp-'") + .append(dayOld) .append(" days');") .toString(); } diff --git a/here-naksha-lib-psql/src/main/resources/naksha_ext.sql b/here-naksha-lib-psql/src/main/resources/naksha_ext.sql index c8a6b7b6a..2ecea52e7 100644 --- a/here-naksha-lib-psql/src/main/resources/naksha_ext.sql +++ b/here-naksha-lib-psql/src/main/resources/naksha_ext.sql @@ -1821,3 +1821,21 @@ BEGIN PERFORM __naksha_create_tx_table(); END $BODY$; + +-- Drop the partition table for the given date. +CREATE OR REPLACE FUNCTION __naksha_delete_hst_partition_for_day(collection text, from_ts timestamptz) + RETURNS void + LANGUAGE 'plpgsql' VOLATILE +AS $BODY$ +DECLARE + sql text; + from_day text; + hst_part_name text; +BEGIN + from_day := to_char(from_ts, 'YYYY_MM_DD'); + hst_part_name := format('%s_hst_%s', collection, from_day); -- example: foo_hst_2023_03_01 + + sql := format('DROP TABLE IF EXISTS %I;', hst_part_name); + EXECUTE sql; +END +$BODY$; \ No newline at end of file From 8f2394b221e2b4dfca004843b38d70fe00d9e8f3 Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Wed, 2 Aug 2023 16:06:57 +0200 Subject: [PATCH 5/6] [MCPODS-6074] fix sql syntax --- .../src/main/java/com/here/naksha/lib/psql/PsqlStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java index 8eb73dba6..1ee51c443 100644 --- a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java +++ b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java @@ -253,7 +253,7 @@ private String createPartitionOfOneDay(int dayPlus) { .append(NakshaAdminCollection.STORAGES.getId()) .append("',current_timestamp+'") .append(dayPlus) - .append(" days');") + .append(" day'::interval);") .toString(); } @@ -265,7 +265,7 @@ private String deletePartitionOfOneDay(int dayOld) { .append(NakshaAdminCollection.STORAGES.getId()) .append("',current_timestamp-'") .append(dayOld) - .append(" days');") + .append(" day'::interval);") .toString(); } From 12a7f318b1ead013064ebb7f0dfe1dd8a6247a8e Mon Sep 17 00:00:00 2001 From: Phuc Mai Date: Wed, 2 Aug 2023 16:35:02 +0200 Subject: [PATCH 6/6] [MCPODS-6074] commit maintain change --- .../src/main/java/com/here/naksha/lib/psql/PsqlStorage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java index 1ee51c443..58c09ca3e 100644 --- a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java +++ b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java @@ -240,6 +240,7 @@ public void maintain() { stmt.execute(createPartitionOfOneDay(2)); stmt.execute(deletePartitionOfOneDay(30)); } + conn.commit(); } catch (Throwable t) { throw unchecked(t); }