MailSet support (static mail listIds)

In order to allow importing of mails we replace legacy MailFolders
(non-static mail listIds) with new MailSets (static mail listIds).
From now on, mails have static mail listIds and static mail elementIds.
To move mails between new MailSets we introduce MailSetEntries
("entries" property on a MailSet), which are index entries sorted by
the received date of the referenced mails (customId). This commit adds
support for new MailSets, while still supporting legacy MailFolders
(mail lists) to support migrating gradually.

* TutanotaModelV74 adds:
  * MailSet support
  * and defaultAlarmList on GroupSettings

* SystemModelV107 adds model changes for counter (unread mails) updates

* Adapt mail list to show MailSet and legacy mails
  The list model is now largely unaware about listIds since it can
  display mails from multiple MailBags. MailBags are static mailLists
  from which a mail is only removed from when the mail is permanently
  deleted.

* Adapt offline storage for mail sets
  Offline storage gained the ability to provide cached entities
  from a list of ids.

packages/tutanota-utils/lib/ArrayUtils.ts

@@ -560,3 +560,26 @@ export function zeroOut(...arrays: (Uint8Array | Int8Array)[]) {
 		a.fill(0)
 	}
 }
+
+/**
+ * @return 1 if first is bigger than second, -1 if second is bigger than first and 0 otherwise
+ */
+export function compare(first: Uint8Array, second: Uint8Array): number {
+	if (first.length > second.length) {
+		return 1
+	} else if (first.length < second.length) {
+		return -1
+	}
+
+	for (let i = 0; i < first.length; i++) {
+		const a = first[i]
+		const b = second[i]
+		if (a > b) {
+			return 1
+		} else if (a < b) {
+			return -1
+		}
+	}
+
+	return 0
+}

packages/tutanota-utils/lib/index.ts

@@ -42,6 +42,7 @@ export {
 	arrayOf,
 	count,
 	zeroOut,
+	compare,
 } from "./ArrayUtils.js"
 export { AsyncResult } from "./AsyncResult.js"
 export { intersection, trisectingDiff, setAddAll, max, maxBy, findBy, min, minBy, mapWith, mapWithout, setEquals, setMap } from "./CollectionUtils.js"

packages/tutanota-utils/test/ArrayUtilsTest.ts

@@ -18,6 +18,7 @@ import {
 	splitInChunks,
 	symmetricDifference,
 } from "../lib/index.js"
+import { compare } from "../lib/ArrayUtils.js"
 
 type ObjectWithId = {
 	v: number
@@ -875,4 +876,18 @@ o.spec("array utils", function () {
 
 		o(arrayOf(2, (idx) => idx + 1 + " one thousand")).deepEquals(["1 one thousand", "2 one thousand"])
 	})
+
+	o("customId comparision", function () {
+		o(compare(new Uint8Array([]), new Uint8Array([]))).equals(0)
+
+		o(compare(new Uint8Array([1]), new Uint8Array([]))).equals(1)
+
+		o(compare(new Uint8Array([]), new Uint8Array([1]))).equals(-1)
+
+		o(compare(new Uint8Array([1, 1]), new Uint8Array([1, 1]))).equals(0)
+
+		o(compare(new Uint8Array([1, 1, 3]), new Uint8Array([1, 1, 2]))).equals(1)
+
+		o(compare(new Uint8Array([1, 1, 2]), new Uint8Array([1, 1, 3]))).equals(-1)
+	})
 })

schemas/sys.json

@@ -380,6 +380,16 @@
 					"info": "AddAssociation GroupKeyRotationData/groupMembershipUpdateData/AGGREGATION/2432."
 				}
 			]
+		},
+		{
+			"version": 108,
+			"changes": [
+				{
+					"name": "RenameAttribute",
+					"sourceType": "WebsocketCounterValue",
+					"info": "RenameAttribute WebsocketCounterValue: mailListId -> counterId."
+				}
+			]
 		}
 	]
 }

schemas/tutanota.json

@@ -450,6 +450,51 @@
 					"info": "RemoveValue Contact/autoTransmitPassword/78."
 				}
 			]
+		},
+		{
+			"version": 74,
+			"changes": [
+				{
+					"name": "AddAssociation",
+					"sourceType": "GroupSettings",
+					"info": "AddAssociation GroupSettings/defaultAlarmsList/AGGREGATION/1446."
+				}
+			]
+		},
+		{
+			"version": 75,
+			"changes": [
+				{
+					"name": "AddValue",
+					"sourceType": "MailFolder",
+					"info": "AddValue MailFolder/isLabel/1454."
+				},
+				{
+					"name": "AddValue",
+					"sourceType": "MailFolder",
+					"info": "AddValue MailFolder/isMailSet/1455."
+				},
+				{
+					"name": "AddAssociation",
+					"sourceType": "MailFolder",
+					"info": "AddAssociation MailFolder/entries/LIST_ASSOCIATION/1456."
+				},
+				{
+					"name": "AddAssociation",
+					"sourceType": "MailBox",
+					"info": "AddAssociation MailBox/archivedMailBags/AGGREGATION/1460."
+				},
+				{
+					"name": "AddAssociation",
+					"sourceType": "MailBox",
+					"info": "AddAssociation MailBox/currentMailBag/AGGREGATION/1461."
+				},
+				{
+					"name": "AddAssociation",
+					"sourceType": "Mail",
+					"info": "AddAssociation Mail/sets/LIST_ELEMENT_ASSOCIATION/1462."
+				}
+			]
 		}
 	]
 }

src/calendar-app/calendar/model/CalendarModel.ts

@@ -220,6 +220,7 @@ export class CalendarModel {
 				group: group._id,
 				color: color,
 				name: null,
+				defaultAlarmsList: [],
 			})
 			userSettingsGroupRoot.groupSettings.push(newGroupSettings)
 			await this.entityClient.update(userSettingsGroupRoot)

src/calendar-app/calendar/search/model/CalendarSearchModel.ts

@@ -133,7 +133,7 @@ export class CalendarSearchModel {
 							continue
 						}
 
-						if (restriction.listIds.length > 0 && !restriction.listIds.includes(listIdPart(event._id))) {
+						if (restriction.folderIds.length > 0 && !restriction.folderIds.includes(listIdPart(event._id))) {
 							// check that the event is in the searched calendar.
 							continue
 						}
@@ -221,7 +221,7 @@ export function isSameSearchRestriction(a: SearchRestriction, b: SearchRestricti
 		a.end === b.end &&
 		isSameAttributeIds &&
 		(a.eventSeries === b.eventSeries || (a.eventSeries === null && b.eventSeries === true) || (a.eventSeries === true && b.eventSeries === null)) &&
-		arrayEquals(a.listIds, b.listIds)
+		arrayEquals(a.folderIds, b.folderIds)
 	)
 }
 

src/calendar-app/calendar/search/model/SearchUtils.ts

@@ -50,8 +50,8 @@ export function getSearchUrl(
 	if (restriction.end) {
 		params.end = restriction.end
 	}
-	if (restriction.listIds.length > 0) {
-		params.list = restriction.listIds
+	if (restriction.folderIds.length > 0) {
+		params.folder = restriction.folderIds
 	}
 
 	if (restriction.eventSeries != null) {
@@ -67,14 +67,14 @@ export function getSearchUrl(
 /**
  * Adjusts the restriction according to the account type if necessary
  */
-export function createRestriction(start: number | null, end: number | null, listIds: Array<string>, eventSeries: boolean): SearchRestriction {
+export function createRestriction(start: number | null, end: number | null, folderIds: Array<string>, eventSeries: boolean): SearchRestriction {
 	return {
 		type: CalendarEventTypeRef,
 		start: start,
 		end: end,
 		field: null,
 		attributeIds: null,
-		listIds,
+		folderIds,
 		eventSeries,
 	}
 }
@@ -85,7 +85,7 @@ export function createRestriction(start: number | null, end: number | null, list
 export function getRestriction(route: string): SearchRestriction {
 	let start: number | null = null
 	let end: number | null = null
-	let listIds: Array<string> = []
+	let folderIds: Array<string> = []
 	let eventSeries: boolean = true
 
 	if (route.startsWith("/calendar") || route.startsWith("/search/calendar")) {
@@ -104,9 +104,9 @@ export function getRestriction(route: string): SearchRestriction {
 				end = filterInt(params["end"])
 			}
 
-			const list = params["list"]
-			if (Array.isArray(list)) {
-				listIds = list
+			const folder = params["folder"]
+			if (Array.isArray(folder)) {
+				folderIds = folder
 			}
 		} catch (e) {
 			console.log("invalid query: " + route, e)
@@ -127,7 +127,7 @@ export function getRestriction(route: string): SearchRestriction {
 		throw new Error("invalid type " + route)
 	}
 
-	return createRestriction(start, end, listIds, eventSeries)
+	return createRestriction(start, end, folderIds, eventSeries)
 }
 
 export function decodeCalendarSearchKey(searchKey: string): { id: Id; start: number } {

src/calendar-app/calendar/search/view/CalendarSearchViewModel.ts

@@ -21,7 +21,7 @@ import {
 	ofClass,
 	TypeRef,
 } from "@tutao/tutanota-utils"
-import { areResultsForTheSameQuery, hasMoreResults, isSameSearchRestriction, CalendarSearchModel } from "../model/CalendarSearchModel.js"
+import { areResultsForTheSameQuery, CalendarSearchModel, hasMoreResults, isSameSearchRestriction } from "../model/CalendarSearchModel.js"
 import { NotFoundError } from "../../../../common/api/common/error/RestError.js"
 import { createRestriction, decodeCalendarSearchKey, encodeCalendarSearchKey, getRestriction } from "../model/SearchUtils.js"
 import Stream from "mithril/stream"
@@ -159,7 +159,7 @@ export class CalendarSearchViewModel {
 	}
 
 	private listIdMatchesRestriction(listId: string, restriction: SearchRestriction): boolean {
-		return restriction.listIds.length === 0 || restriction.listIds.includes(listId)
+		return restriction.folderIds.length === 0 || restriction.folderIds.includes(listId)
 	}
 
 	onNewUrl(args: Record<string, any>, requestedPath: string) {
@@ -217,7 +217,7 @@ export class CalendarSearchViewModel {
 
 		this.startDate = restriction.start ? new Date(restriction.start) : null
 		this.endDate = restriction.end ? new Date(restriction.end) : null
-		this.selectedCalendar = this.extractCalendarListIds(restriction.listIds)
+		this.selectedCalendar = this.extractCalendarListIds(restriction.folderIds)
 		this.includeRepeatingEvents = restriction.eventSeries ?? true
 		this.lazyCalendarInfos.load()
 		this.latestCalendarRestriction = restriction
@@ -415,12 +415,12 @@ export class CalendarSearchViewModel {
 
 				return { items: entries, complete }
 			},
-			loadSingle: async (elementId: Id) => {
+			loadSingle: async (_listId: Id, elementId: Id) => {
 				const lastResult = this._searchResult
 				if (!lastResult) {
 					return null
 				}
-				const id = lastResult.results.find((r) => r[1] === elementId)
+				const id = lastResult.results.find((resultId) => elementIdPart(resultId) === elementId)
 				if (id) {
 					return this.entityClient
 						.load(lastResult.restriction.type, id)
@@ -446,7 +446,7 @@ export class CalendarSearchViewModel {
 		if (result && isSameTypeRef(typeRef, result.restriction.type)) {
 			// The list id must be null/empty, otherwise the user is filtering by list, and it shouldn't be ignored
 
-			const ignoreList = isSameTypeRef(typeRef, MailTypeRef) && result.restriction.listIds.length === 0
+			const ignoreList = isSameTypeRef(typeRef, MailTypeRef) && result.restriction.folderIds.length === 0
 
 			return result.results.some((r) => this.compareItemId(r, id, ignoreList))
 		}

src/calendar-app/calendar/view/CalendarView.ts

@@ -802,6 +802,7 @@ export class CalendarView extends BaseTopLevelView implements TopLevelView<Calen
 						group: groupInfo.group,
 						color: properties.color,
 						name: shared && properties.name !== groupInfo.name ? properties.name : null,
+						defaultAlarmsList: [],
 					})
 					userSettingsGroupRoot.groupSettings.push(newGroupSettings)
 				}

src/common/api/common/CommonMailUtils.ts

@@ -1,22 +1,22 @@
 import type { FolderSystem } from "./mail/FolderSystem.js"
 import { Body, Mail, MailFolder } from "../entities/tutanota/TypeRefs.js"
-import { MailFolderType } from "./TutanotaConstants.js"
+import { MailSetKind } from "./TutanotaConstants.js"
 
-export function isSubfolderOfType(system: FolderSystem, folder: MailFolder, type: MailFolderType): boolean {
+export function isSubfolderOfType(system: FolderSystem, folder: MailFolder, type: MailSetKind): boolean {
 	const systemFolder = system.getSystemFolderByType(type)
 	return systemFolder != null && system.checkFolderForAncestor(folder, systemFolder._id)
 }
 
 /**
- * Returns true if given folder is the {@link MailFolderType.SPAM} or {@link MailFolderType.TRASH} folder, or a descendant of those folders.
+ * Returns true if given folder is the {@link MailSetKind.SPAM} or {@link MailSetKind.TRASH} folder, or a descendant of those folders.
  */
 export function isSpamOrTrashFolder(system: FolderSystem, folder: MailFolder): boolean {
 	// not using isOfTypeOrSubfolderOf because checking the type first is cheaper
 	return (
-		folder.folderType === MailFolderType.TRASH ||
-		folder.folderType === MailFolderType.SPAM ||
-		isSubfolderOfType(system, folder, MailFolderType.TRASH) ||
-		isSubfolderOfType(system, folder, MailFolderType.SPAM)
+		folder.folderType === MailSetKind.TRASH ||
+		folder.folderType === MailSetKind.SPAM ||
+		isSubfolderOfType(system, folder, MailSetKind.TRASH) ||
+		isSubfolderOfType(system, folder, MailSetKind.SPAM)
 	)
 }
 

src/common/api/common/TutanotaConstants.ts

@@ -23,7 +23,7 @@ export const REQUEST_SIZE_LIMIT_MAP: Map<string, number> = new Map([
 
 export const SYSTEM_GROUP_MAIL_ADDRESS = "system@tutanota.de"
 
-export const getMailFolderType = (folder: MailFolder): MailFolderType => downcast(folder.folderType)
+export const getMailFolderType = (folder: MailFolder): MailSetKind => downcast(folder.folderType)
 
 type ObjectPropertyKey = string | number | symbol
 export const reverse = <K extends ObjectPropertyKey, V extends ObjectPropertyKey>(objectMap: Record<K, V>): Record<V, K> =>
@@ -84,14 +84,15 @@ export const enum BucketPermissionType {
 	External = "3",
 }
 
-export enum MailFolderType {
+export enum MailSetKind {
 	CUSTOM = "0",
 	INBOX = "1",
 	SENT = "2",
 	TRASH = "3",
 	ARCHIVE = "4",
 	SPAM = "5",
 	DRAFT = "6",
+	ALL = "7",
 }
 
 export const enum ReplyType {