Naksha maintenance

heremaps · Aug 3, 2023 · 7010d3c · 7010d3c
2 parents d30f9ee + 21c7757
commit 7010d3c
Show file tree

Hide file tree

Showing 4 changed files with 219 additions and 3 deletions.
diff --git a/docs/TESTNOTEBOOK.ipynb b/docs/TESTNOTEBOOK.ipynb
@@ -0,0 +1,154 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "This is the initial draft of a Kotlin Notebook, basically the same as Jupyter notebooks, where code snippets can also be included in the markdown docs, and executed. This is more popular for workshops with smaller units of code needing execution, but theoretically there is a setting that allows new notebooks to import project dependencies, so doing something like ramping up Naksha locally should be achievable in a code block in notebooks. Furthermore the ability to render other formats like HTML webpage or LaTeX mathematical formulas directly inside one notebook file might be a good presentation for a product.\n",
+ "\n",
+ "Follow the guide at https://blog.jetbrains.com/kotlin/2023/07/introducing-kotlin-notebook/ to install the plugin for IntelliJ.\n",
+ "\n",
+ "Currently Jetbrains has [this notebook](https://datalore.jetbrains.com/report/static/sBgNeHxfaKG1tVRDWl3KqU/DMtWM05XWOT2Oi6yxZjVkJ) to showcase the capabilities of these Kotlin notebooks."
+ ],
+ "metadata": {
+ "collapsed": false
+ }
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "To setup a Naksha Hub server on your local machine, first define the main Postgres password:"
+ ],
+ "metadata": {
+ "collapsed": false
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "outputs": [],
+ "source": [
+ "var password = \"postgres\""
+ ],
+ "metadata": {
+ "collapsed": false,
+ "ExecuteTime": {
+ "end_time": "2023-07-28T12:10:48.605019065Z",
+ "start_time": "2023-07-28T12:10:48.493613134Z"
+ }
+ }
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "Then run the code block below to start the Hub:"
+ ],
+ "metadata": {
+ "collapsed": false
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n",
+ "com.here.naksha.lib.core.exceptions.UncheckedException: java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n",
+ "\tat com.here.naksha.lib.core.exceptions.UncheckedException.unchecked(UncheckedException.java:43)\n",
+ "\tat com.here.naksha.lib.psql.PsqlStorage.init(PsqlStorage.java:195)\n",
+ "\tat com.here.naksha.app.service.NakshaHub.<init>(NakshaHub.java:168)\n",
+ "\tat com.here.naksha.app.service.NakshaHub.newHub(NakshaHub.java:130)\n",
+ "\tat Line_3_jupyter.<init>(Line_3.jupyter.kts:4)\n",
+ "\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n",
+ "\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)\n",
+ "\tat java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n",
+ "\tat java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)\n",
+ "\tat java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)\n",
+ "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.evalWithConfigAndOtherScriptsResults(BasicJvmScriptEvaluator.kt:105)\n",
+ "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke$suspendImpl(BasicJvmScriptEvaluator.kt:47)\n",
+ "\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke(BasicJvmScriptEvaluator.kt)\n",
+ "\tat kotlin.script.experimental.jvm.BasicJvmReplEvaluator.eval(BasicJvmReplEvaluator.kt:49)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl$eval$resultWithDiagnostics$1.invokeSuspend(InternalEvaluatorImpl.kt:103)\n",
+ "\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)\n",
+ "\tat kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:106)\n",
+ "\tat kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:284)\n",
+ "\tat kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:85)\n",
+ "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:59)\n",
+ "\tat kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)\n",
+ "\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:38)\n",
+ "\tat kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl.eval(InternalEvaluatorImpl.kt:103)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:75)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:73)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.withHost(repl.kt:666)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl.execute(CellExecutorImpl.kt:73)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.repl.CellExecutor$DefaultImpls.execute$default(CellExecutor.kt:15)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl$evalEx$1.invoke(repl.kt:479)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl$evalEx$1.invoke(repl.kt:470)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.withEvalContext(repl.kt:433)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.ReplForJupyterImpl.evalEx(repl.kt:470)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.messaging.ProtocolKt$shellMessagesHandler$2$res$1.invoke(protocol.kt:318)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.messaging.ProtocolKt$shellMessagesHandler$2$res$1.invoke(protocol.kt:312)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.JupyterExecutorImpl$runExecution$execThread$1.invoke(execution.kt:37)\n",
+ "\tat org.jetbrains.kotlinx.jupyter.JupyterExecutorImpl$runExecution$execThread$1.invoke(execution.kt:32)\n",
+ "\tat kotlin.concurrent.ThreadsKt$thread$thread$1.run(Thread.kt:30)\n",
+ "Caused by: java.lang.NoSuchMethodError: 'org.slf4j.spi.LoggingEventBuilder com.here.naksha.lib.core.NakshaLogger.atInfo()'\n",
+ "\tat com.here.naksha.lib.core.NakshaLogger.atInfo(NakshaLogger.java:415)\n",
+ "\tat com.here.naksha.lib.psql.PsqlStorage.init(PsqlStorage.java:173)\n",
+ "\t... 36 more\n"
+ ]
+ }
+ ],
+ "source": [
+ "import com.here.naksha.app.service.NakshaHub\n",
+ "import java.util.concurrent.TimeUnit\n",
+ "\n",
+ "var hub: NakshaHub? = NakshaHub.newHub(\n",
+ " \"jdbc:postgresql://localhost/postgres?user=postgres&password=$password\",\n",
+ " \"local\"\n",
+ ")\n",
+ "hub!!.start()\n",
+ "hub!!.join(TimeUnit.SECONDS.toMillis(60))\n"
+ ],
+ "metadata": {
+ "collapsed": false,
+ "ExecuteTime": {
+ "end_time": "2023-07-28T12:12:46.202817846Z",
+ "start_time": "2023-07-28T12:12:45.881557242Z"
+ }
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [],
+ "metadata": {
+ "collapsed": false
+ }
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Kotlin",
+ "language": "kotlin",
+ "name": "kotlin"
+ },
+ "language_info": {
+ "name": "kotlin",
+ "version": "1.8.20",
+ "mimetype": "text/x-kotlin",
+ "file_extension": ".kt",
+ "pygments_lexer": "kotlin",
+ "codemirror_mode": "text/x-kotlin",
+ "nbconvert_exporter": ""
+ },
+ "ktnbPluginMetadata": {
+ "isBuildProject": true
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java b/here-naksha-app-service/src/test/java/com/here/naksha/app/service/NakshaHubTest.java
@@ -28,7 +28,11 @@ class NakshaHubTest {
 
  @BeforeAll
  static void prepare() {
- hub = newHub("jdbc:postgresql://localhost/postgres?user=postgres&password=password", "local");
+ String password = System.getenv("TEST_NAKSHA_PSQL_PASS");
+ if (password == null) password = "password";
+ hub = newHub(
+ "jdbc:postgresql://localhost/postgres?user=postgres&password=" + password,
+ "local"); // this string concat only happens once, its ok ;)
  hub.start();
  }
 
@@ -44,6 +48,6 @@ void startup() throws InterruptedException {
  static void close() throws InterruptedException {
  // TODO: Find a way to gracefully shutdown the server.
  // To do some manual testing with the running service, uncomment this:
- //hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(60));
+ //hub.join(java.util.concurrent.TimeUnit.SECONDS.toMillis(10));
  }
 }
diff --git a/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java b/here-naksha-lib-psql/src/main/java/com/here/naksha/lib/psql/PsqlStorage.java
@@ -22,6 +22,7 @@
 import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked;
 import static com.here.naksha.lib.psql.SQL.escapeId;
 
+import com.here.naksha.lib.core.NakshaAdminCollection;
 import com.here.naksha.lib.core.NakshaVersion;
 import com.here.naksha.lib.core.lambdas.Pe1;
 import com.here.naksha.lib.core.models.TxSignalSet;
@@ -224,7 +225,46 @@ public void init() {
  */
  @Override
  public void maintain() {
- // throw new UnsupportedOperationException("maintain");
+ try (final Connection conn = dataSource.getConnection()) {
+ try (final Statement stmt = conn.createStatement()) {
+ stmt.execute(new StringBuilder()
+ .append("SET SESSION search_path TO \"")
+ .append(getSchema())
+ .append("\",\"public\",\"topology\";")
+ .toString());
+ stmt.execute(createPartitionOfOneDay(0));
+ stmt.execute(createPartitionOfOneDay(1));
+ stmt.execute(createPartitionOfOneDay(2));
+ stmt.execute(deletePartitionOfOneDay(30));
+ }
+ conn.commit();
+ } catch (Throwable t) {
+ throw unchecked(t);
+ }
+ }
+
+ private String createPartitionOfOneDay(int dayPlus) {
+ return new StringBuilder()
+ .append("SELECT ")
+ .append(getSchema())
+ .append(".__naksha_create_hst_partition_for_day('")
+ .append(NakshaAdminCollection.STORAGES.getId())
+ .append("',current_timestamp+'")
+ .append(dayPlus)
+ .append(" day'::interval);")
+ .toString();
+ }
+
+ private String deletePartitionOfOneDay(int dayOld) {
+ return new StringBuilder()
+ .append("SELECT ")
+ .append(getSchema())
+ .append(".__naksha_delete_hst_partition_for_day('")
+ .append(NakshaAdminCollection.STORAGES.getId())
+ .append("',current_timestamp-'")
+ .append(dayOld)
+ .append(" day'::interval);")
+ .toString();
  }
 
  /**

diff --git a/here-naksha-lib-psql/src/main/resources/naksha_ext.sql b/here-naksha-lib-psql/src/main/resources/naksha_ext.sql
@@ -1821,3 +1821,21 @@ BEGIN
  PERFORM __naksha_create_tx_table();
 END
 $BODY$;
+
+-- Drop the partition table for the given date.
+CREATE OR REPLACE FUNCTION __naksha_delete_hst_partition_for_day(collection text, from_ts timestamptz)
+ RETURNS void
+ LANGUAGE 'plpgsql' VOLATILE
+AS $BODY$
+DECLARE
+ sql text;
+ from_day text;
+ hst_part_name text;
+BEGIN
+ from_day := to_char(from_ts, 'YYYY_MM_DD');
+ hst_part_name := format('%s_hst_%s', collection, from_day); -- example: foo_hst_2023_03_01
+
+ sql := format('DROP TABLE IF EXISTS %I;', hst_part_name);
+ EXECUTE sql;
+END
+$BODY$;