From e9760406f17fd34c47f41d125ddbdea03f7b2496 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:14:20 +0200 Subject: [PATCH 01/15] add include_tables to the config --- config.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index f1d84c0..ca976de 100644 --- a/config.py +++ b/config.py @@ -89,6 +89,11 @@ def validate_config(config): "Config error: Name of the Mergin Maps project should be provided in the namespace/name format." ) + if "skip_tables" in conn and "include_tables" in conn: + raise ConfigError( + "Config error: `skip_tables` and `include_tables` cannot both be set for the same connection." + ) + if "skip_tables" in conn: if conn.skip_tables is None: continue @@ -101,7 +106,21 @@ def validate_config(config): conn.skip_tables, list, ): - raise ConfigError("Config error: Ignored tables parameter should be a list") + raise ConfigError("Config error: `skip_tables` parameter should be a list") + + if "include_tables" in conn: + if conn.include_tables is None: + continue + elif isinstance( + conn.include_tables, + str, + ): + continue + elif not isinstance( + conn.include_tables, + list, + ): + raise ConfigError("Config error: `include_tables` parameter should be a list") if "notification" in config: settings = [ From 710a5dcfd5754045e7bddceef54e9cf1db6c4fe4 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:15:06 +0200 Subject: [PATCH 02/15] create get_include_tables() and make it shared logic with get_ignored_tables() --- config.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index ca976de..6b39550 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ import smtplib import subprocess import tempfile +import typing from dynaconf import Dynaconf import dynaconf @@ -163,26 +164,35 @@ def validate_config(config): raise ConfigError(f"Config SMTP Error: {err}.") +def _get_tables(tables: typing.Union[None, str, typing.List, dynaconf.vendor.box.box_list.BoxList], param_name: str): + if tables is None: + return [] + elif isinstance(tables, str): + return [tables] + elif isinstance(tables, list): + if len(tables) < 1: + return [] + elif isinstance(tables, dynaconf.vendor.box.box_list.BoxList): + return tables.to_list() + return tables + else: + raise ConfigError(f"Config error: `{param_name}` parameter should be a list or a string.") + + def get_ignored_tables( connection, ): if "skip_tables" in connection: - if connection.skip_tables is None: - return [] - elif isinstance( - connection.skip_tables, - str, - ): - return [connection.skip_tables] - elif isinstance( - connection.skip_tables, - list, - ): - if len(connection.skip_tables) < 1: - return [] - elif isinstance(connection.skip_tables, dynaconf.vendor.box.box_list.BoxList): - return connection.skip_tables.to_list() - return connection.skip_tables + return _get_tables(connection.skip_tables, "skip_tables") + else: + return [] + + +def get_include_tables( + connection, +): + if "include_tables" in connection: + return _get_tables(connection.include_tables, "include_tables") else: return [] From 9f21d5e5098fab5e2b2cd67db131b8ee14d74016 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:15:53 +0200 Subject: [PATCH 03/15] add tests --- test/conftest.py | 10 +++++++- test/test_basic.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ test/test_config.py | 46 +++++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 49f9af5..aa7569c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -80,7 +80,9 @@ def cleanup_db( cur.execute("COMMIT") -def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables=[], *extra_init_files): +def init_sync_from_geopackage( + mc, project_name, source_gpkg_path, ignored_tables=[], include_tables=None, *extra_init_files +): """ Initialize sync from given GeoPackage file: - (re)create Mergin Maps project with the file @@ -152,6 +154,12 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables elif isinstance(ignored_tables, list): connection["skip_tables"] = ignored_tables + if include_tables: + if isinstance(include_tables, str): + connection["include_tables"] = [include_tables] + elif isinstance(include_tables, list): + connection["include_tables"] = include_tables + config.update( { "GEODIFF_EXE": GEODIFF_EXE, diff --git a/test/test_basic.py b/test/test_basic.py index 64220a3..b5d31f3 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -1005,3 +1005,64 @@ def test_dbsync_clean_from_gpkg( dbsync_init(mc) dbsync_pull(mc) dbsync_push(mc) + + +def test_init_with_include_tables( + mc: MerginClient, +): + project_name = "test_init_include_tables" + source_gpkg_path = os.path.join( + TEST_DATA_DIR, + "base_2tables.gpkg", + ) + project_dir = os.path.join( + TMP_DIR, + project_name + "_work", + ) + db_schema_main = project_name + "_main" + + init_sync_from_geopackage( + mc, + project_name, + source_gpkg_path, + include_tables=["points"], + ) + + # only points should exist in the main schema, lines should be absent + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + cur.execute( + sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format( + sql.Identifier(db_schema_main) + ) + ) + assert cur.fetchone()[0] == False + + cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) + assert cur.fetchone()[0] == 0 + + # run init again, nothing should change + dbsync_init(mc) + cur.execute( + sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format( + sql.Identifier(db_schema_main) + ) + ) + assert cur.fetchone()[0] == False + + # push a change that touches both tables, only points should be pulled + shutil.copy( + os.path.join(TEST_DATA_DIR, "modified_all.gpkg"), + os.path.join(project_dir, "test_sync.gpkg"), + ) + mc.push_project(project_dir) + + dbsync_pull(mc) + cur.execute( + sql.SQL("SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '{}' AND tablename = 'lines');").format( + sql.Identifier(db_schema_main) + ) + ) + assert cur.fetchone()[0] == False + cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) + assert cur.fetchone()[0] == 4 diff --git a/test/test_config.py b/test/test_config.py index 8d0258e..0275bf7 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -8,7 +8,7 @@ import pytest -from config import ConfigError, config, get_ignored_tables, validate_config +from config import ConfigError, config, get_ignored_tables, get_include_tables, validate_config from .conftest import _reset_config @@ -345,3 +345,47 @@ def test_config_notification_setup(): with pytest.raises(ConfigError, match="Config SMTP Error"): validate_config(config) + + +def test_include_tables(): + _reset_config() + base = dict(config.connections[0]) + + for value in (None, [], "table", ["table"]): + config.update({"CONNECTIONS": [{**base, "include_tables": value}]}) + validate_config(config) + + # invalid type + config.update({"CONNECTIONS": [{**base, "include_tables": 42}]}) + with pytest.raises(ConfigError, match="`include_tables` parameter should be a list"): + validate_config(config) + + +def test_get_include_tables(): + _reset_config() + base = dict(config.connections[0]) + + config.update({"CONNECTIONS": [{**base, "include_tables": None}]}) + assert get_include_tables(config.connections[0]) == [] + + config.update({"CONNECTIONS": [{**base, "include_tables": []}]}) + assert get_include_tables(config.connections[0]) == [] + + config.update({"CONNECTIONS": [{**base, "include_tables": "table"}]}) + assert get_include_tables(config.connections[0]) == ["table"] + + config.update({"CONNECTIONS": [{**base, "include_tables": ["table"]}]}) + assert get_include_tables(config.connections[0]) == ["table"] + + # connection without include_tables configured + config.update({"CONNECTIONS": [base]}) + assert get_include_tables(config.connections[0]) == [] + + +def test_skip_and_include_tables_mutually_exclusive(): + _reset_config() + base = dict(config.connections[0]) + + config.update({"CONNECTIONS": [{**base, "skip_tables": ["a"], "include_tables": ["b"]}]}) + with pytest.raises(ConfigError, match="cannot both be set"): + validate_config(config) From ea35d9ad51f77d955605839eaea82eda40d21e4c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:16:20 +0200 Subject: [PATCH 04/15] update using --- docs/using.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/using.md b/docs/using.md index 2d83b6e..a058734 100644 --- a/docs/using.md +++ b/docs/using.md @@ -78,11 +78,15 @@ daemon: - `--test-notification-email` used to test send notification email (see below for details about sending emails in case of sync fails) -## Excluding tables from sync +## Selecting which tables are synced -Sometimes in the database there are tables that should not be synchronised to Mergin Maps projects. It is possible to ignore -these tables and not sync them. To do so add `skip_tables` setting to the corresponding `connections` entry in the config -file: +Sometimes in the database there are tables that should not be synchronised to Mergin Maps projects. There are two +mutually exclusive ways to control which tables get synced. Use **only one** of them per connection - setting both +`skip_tables` and `include_tables` for the same connection is a configuration error. + +### Excluding tables (`skip_tables`) + +Add `skip_tables` to the corresponding `connections` entry to ignore the listed tables and sync everything else: ```yaml connections: @@ -95,6 +99,22 @@ connections: - table2 ``` +### Including only specific tables (`include_tables`) + +Alternatively, add `include_tables` to sync **only** the listed tables and ignore everything else. This is useful when +a database contains many tables but only a few should be synced: + +```yaml +connections: + - driver: postgres + # ... + mergin_project: john/myproject + sync_file: sync.gpkg + include_tables: + - table1 + - table2 +``` + ## Email notifications on sync failures To simplify db-sync monitoring, it is possible to set up notification emails when a sync failure happens. Simply add `notification` section in the configuration file as described below. From 6af216a91421666d546b651cd549712c58c4b6fc Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:18:21 +0200 Subject: [PATCH 05/15] create a function to create args for geodiff --- dbsync.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dbsync.py b/dbsync.py index 1e11654..12bb260 100644 --- a/dbsync.py +++ b/dbsync.py @@ -42,6 +42,7 @@ config, validate_config, get_ignored_tables, + get_include_tables, ConfigError, ) @@ -96,6 +97,23 @@ def _tables_list_to_string( return ";".join(tables) +def _tables_filter_args( + ignored_tables, + include_tables, +): + """Build the geodiff CLI args for table filtering. + + ``skip_tables`` and ``include_tables`` are mutually exclusive (validated in + ``validate_config``), so at most one of them is ever set. Returns an empty + list when no filtering is configured. + """ + if include_tables: + return ["--include-tables", _tables_list_to_string(include_tables)] + if ignored_tables: + return ["--skip-tables", _tables_list_to_string(ignored_tables)] + return [] + + def _check_has_working_dir( work_path, ): From 84d9e4086b67bedaba907a99f75ac8982ce12b3f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:19:54 +0200 Subject: [PATCH 06/15] allow passing to included_tables to geodiff --- dbsync.py | 227 ++++++++++++++++++------------------------------------ 1 file changed, 77 insertions(+), 150 deletions(-) diff --git a/dbsync.py b/dbsync.py index 12bb260..f2c7a90 100644 --- a/dbsync.py +++ b/dbsync.py @@ -217,35 +217,21 @@ def _geodiff_create_changeset( modified, changeset, ignored_tables, + include_tables, ): - if ignored_tables: - _run_geodiff( - [ - config.geodiff_exe, - "diff", - "--driver", - driver, - conn_info, - "--skip-tables", - _tables_list_to_string(ignored_tables), - base, - modified, - changeset, - ] - ) - else: - _run_geodiff( - [ - config.geodiff_exe, - "diff", - "--driver", - driver, - conn_info, - base, - modified, - changeset, - ] - ) + _run_geodiff( + [ + config.geodiff_exe, + "diff", + "--driver", + driver, + conn_info, + *_tables_filter_args(ignored_tables, include_tables), + base, + modified, + changeset, + ] + ) def _geodiff_apply_changeset( @@ -254,33 +240,20 @@ def _geodiff_apply_changeset( base, changeset, ignored_tables, + include_tables, ): - if ignored_tables: - _run_geodiff( - [ - config.geodiff_exe, - "apply", - "--driver", - driver, - conn_info, - "--skip-tables", - _tables_list_to_string(ignored_tables), - base, - changeset, - ] - ) - else: - _run_geodiff( - [ - config.geodiff_exe, - "apply", - "--driver", - driver, - conn_info, - base, - changeset, - ] - ) + _run_geodiff( + [ + config.geodiff_exe, + "apply", + "--driver", + driver, + conn_info, + *_tables_filter_args(ignored_tables, include_tables), + base, + changeset, + ] + ) def _geodiff_rebase( @@ -291,37 +264,22 @@ def _geodiff_rebase( base2their, conflicts, ignored_tables, + include_tables, ): - if ignored_tables: - _run_geodiff( - [ - config.geodiff_exe, - "rebase-db", - "--driver", - driver, - conn_info, - "--skip-tables", - _tables_list_to_string(ignored_tables), - base, - our, - base2their, - conflicts, - ] - ) - else: - _run_geodiff( - [ - config.geodiff_exe, - "rebase-db", - "--driver", - driver, - conn_info, - base, - our, - base2their, - conflicts, - ] - ) + _run_geodiff( + [ + config.geodiff_exe, + "rebase-db", + "--driver", + driver, + conn_info, + *_tables_filter_args(ignored_tables, include_tables), + base, + our, + base2their, + conflicts, + ] + ) def _geodiff_list_changes_details( @@ -386,39 +344,23 @@ def _geodiff_make_copy( dst_conn_info, dst, ignored_tables, + include_tables, ): - if ignored_tables: - _run_geodiff( - [ - config.geodiff_exe, - "copy", - "--driver-1", - src_driver, - src_conn_info, - "--driver-2", - dst_driver, - dst_conn_info, - "--skip-tables", - _tables_list_to_string(ignored_tables), - src, - dst, - ] - ) - else: - _run_geodiff( - [ - config.geodiff_exe, - "copy", - "--driver-1", - src_driver, - src_conn_info, - "--driver-2", - dst_driver, - dst_conn_info, - src, - dst, - ] - ) + _run_geodiff( + [ + config.geodiff_exe, + "copy", + "--driver-1", + src_driver, + src_conn_info, + "--driver-2", + dst_driver, + dst_conn_info, + *_tables_filter_args(ignored_tables, include_tables), + src, + dst, + ] + ) def _geodiff_create_changeset_dr( @@ -430,41 +372,24 @@ def _geodiff_create_changeset_dr( dst, changeset, ignored_tables, + include_tables, ): - if ignored_tables: - _run_geodiff( - [ - config.geodiff_exe, - "diff", - "--driver-1", - src_driver, - src_conn_info, - "--driver-2", - dst_driver, - dst_conn_info, - "--skip-tables", - _tables_list_to_string(ignored_tables), - src, - dst, - changeset, - ] - ) - else: - _run_geodiff( - [ - config.geodiff_exe, - "diff", - "--driver-1", - src_driver, - src_conn_info, - "--driver-2", - dst_driver, - dst_conn_info, - src, - dst, - changeset, - ] - ) + _run_geodiff( + [ + config.geodiff_exe, + "diff", + "--driver-1", + src_driver, + src_conn_info, + "--driver-2", + dst_driver, + dst_conn_info, + *_tables_filter_args(ignored_tables, include_tables), + src, + dst, + changeset, + ] + ) def _compare_datasets( @@ -475,6 +400,7 @@ def _compare_datasets( dst_conn_info, dst, ignored_tables, + include_tables, summary_only=True, ): """Compare content of two datasets (from various drivers) and return geodiff JSON summary of changes""" @@ -498,6 +424,7 @@ def _compare_datasets( dst, tmp_changeset, ignored_tables, + include_tables, ) if summary_only: return _geodiff_list_changes_summary(tmp_changeset) From 97eaf7f675c05783d1ba38375b22057eb526bbbf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:21:24 +0200 Subject: [PATCH 07/15] use include_tables in geodiff functions --- dbsync.py | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/dbsync.py b/dbsync.py index f2c7a90..779a8df 100644 --- a/dbsync.py +++ b/dbsync.py @@ -675,6 +675,7 @@ def pull(conn_cfg, mc): logging.debug(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") ignored_tables = get_ignored_tables(conn_cfg) + include_tables = get_include_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] work_dir = os.path.join( @@ -690,7 +691,10 @@ def pull(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = _get_mergin_project(work_dir) - mp.set_tables_to_skip(ignored_tables) + if include_tables: + mp.set_tables_to_include(include_tables) + else: + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") @@ -752,6 +756,7 @@ def pull(conn_cfg, mc): conn_cfg.modified, tmp_base2our, ignored_tables, + include_tables, ) needs_rebase = False @@ -779,6 +784,7 @@ def pull(conn_cfg, mc): gpkg_basefile, tmp_base2their, ignored_tables, + include_tables, ) # summarize changes @@ -790,8 +796,8 @@ def pull(conn_cfg, mc): if not needs_rebase: logging.debug("Applying new version [no rebase]") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables, include_tables) else: logging.debug("Applying new version [WITH rebase]") tmp_conflicts = os.path.join(tmp_dir, f"{project_name}-dbsync-pull-conflicts") @@ -803,8 +809,9 @@ def pull(conn_cfg, mc): tmp_base2their, tmp_conflicts, ignored_tables, + include_tables, ) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables) os.remove(gpkg_basefile_old) conn = psycopg2.connect(conn_cfg.conn_info) @@ -822,6 +829,7 @@ def status(conn_cfg, mc): logging.debug(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") ignored_tables = get_ignored_tables(conn_cfg) + include_tables = get_include_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -839,7 +847,10 @@ def status(conn_cfg, mc): # get basic information mp = _get_mergin_project(work_dir) - mp.set_tables_to_skip(ignored_tables) + if include_tables: + mp.set_tables_to_include(include_tables) + else: + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") project_path = mp.project_full_name() @@ -905,6 +916,7 @@ def status(conn_cfg, mc): conn_cfg.modified, tmp_changeset_file, ignored_tables, + include_tables, ) if os.path.getsize(tmp_changeset_file) == 0: @@ -921,6 +933,7 @@ def push(conn_cfg, mc): logging.debug(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") ignored_tables = get_ignored_tables(conn_cfg) + include_tables = get_include_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -944,7 +957,10 @@ def push(conn_cfg, mc): _check_has_sync_file(gpkg_full_path) mp = _get_mergin_project(work_dir) - mp.set_tables_to_skip(ignored_tables) + if include_tables: + mp.set_tables_to_include(include_tables) + else: + mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") @@ -991,6 +1007,7 @@ def push(conn_cfg, mc): conn_cfg.modified, tmp_changeset_file, ignored_tables, + include_tables, ) if os.path.getsize(tmp_changeset_file) == 0: @@ -1003,7 +1020,7 @@ def push(conn_cfg, mc): # write changes to the local geopackage logging.debug("Writing DB changes to working dir...") - _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file, ignored_tables) + _geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file, ignored_tables, include_tables) # write to the server try: @@ -1017,7 +1034,7 @@ def push(conn_cfg, mc): # update base schema in the DB logging.debug("Updating DB base schema...") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables) + _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables, include_tables) _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version) @@ -1030,6 +1047,7 @@ def init( logging.debug(f"Processing Mergin Maps project '{conn_cfg.mergin_project}'") ignored_tables = get_ignored_tables(conn_cfg) + include_tables = get_include_tables(conn_cfg) project_name = conn_cfg.mergin_project.split("/")[1] @@ -1085,6 +1103,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, summary_only=False, ) changes = json.dumps(changes_gpkg_base, indent=2) @@ -1166,6 +1185,7 @@ def init( conn_cfg.conn_info, conn_cfg.modified, ignored_tables, + include_tables, ) logging.debug("Checking 'base' schema content...") summary_base = _compare_datasets( @@ -1176,6 +1196,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, ) if len(summary_base): # seems someone modified base schema manually - this should never happen! @@ -1219,6 +1240,7 @@ def init( conn_cfg.conn_info, conn_cfg.modified, ignored_tables, + include_tables, ) # COPY: modified -> base @@ -1230,6 +1252,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, ) # sanity check to verify that right after initialization we do not have any changes @@ -1243,6 +1266,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, summary_only=False, ) # mark project version into db schema @@ -1285,6 +1309,7 @@ def init( "", gpkg_full_path, ignored_tables, + include_tables, ) logging.debug("Checking 'base' schema content...") summary_base = _compare_datasets( @@ -1295,6 +1320,7 @@ def init( "", gpkg_full_path, ignored_tables, + include_tables, ) if len(summary_base): logging.debug( @@ -1338,6 +1364,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, ) # COPY: modified -> gpkg @@ -1349,6 +1376,7 @@ def init( "", gpkg_full_path, ignored_tables, + include_tables, ) # sanity check to verify that right after initialization we do not have any changes @@ -1362,6 +1390,7 @@ def init( conn_cfg.conn_info, conn_cfg.base, ignored_tables, + include_tables, summary_only=False, ) if len(changes_gpkg_base): From 0a8b66db4d40142d373a41c5151eddbd6c5a94fc Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 25 Jun 2026 17:24:09 +0200 Subject: [PATCH 08/15] bump version --- CHANGELOG.md | 4 ++++ version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2066d4d..15375ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.3.0 + +- Add `include_tables` connection option to sync only the listed tables (mutually exclusive with `skip_tables`) + ## 2.2.0 - Add support for constraints (geodiff 2.1.0) diff --git a/version.py b/version.py index 8a124bf..55e4709 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -__version__ = "2.2.0" +__version__ = "2.3.0" From 5724d77bbce1cc21125e61cb0871e03fda6b9e63 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 08:47:13 +0200 Subject: [PATCH 09/15] notification should not be in basic settings, it is added in the test cases where it is needed --- test/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index aa7569c..6db288d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -50,6 +50,9 @@ def _reset_config(project_name: str = "mergin", init_from: str = "gpkg"): } ) + if "NOTIFICATION" in config: + config.unset("NOTIFICATION", force=True) + def cleanup( mc: MerginClient, From 21728f1120b578cc66096d4b93a26f8895f63168 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 08:48:04 +0200 Subject: [PATCH 10/15] add new required parameter --- dbsync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index 779a8df..a3e57d3 100644 --- a/dbsync.py +++ b/dbsync.py @@ -880,7 +880,7 @@ def status(conn_cfg, mc): logging.debug("") logging.debug("Server is at version " + server_info["version"]) - status_pull = mp.get_pull_changes(server_info["files"]) + status_pull = mp.get_pull_changes(server_info["files"], server_info["version"]) if status_pull["added"] or status_pull["updated"] or status_pull["removed"]: logging.debug("There are pending changes on server:") _print_mergin_changes(status_pull) From 5fb80ac3992093151b34a42c265e2da4f0df512b Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 08:48:11 +0200 Subject: [PATCH 11/15] restyle --- dbsync.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dbsync.py b/dbsync.py index a3e57d3..1214e21 100644 --- a/dbsync.py +++ b/dbsync.py @@ -796,8 +796,12 @@ def pull(conn_cfg, mc): if not needs_rebase: logging.debug("Applying new version [no rebase]") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables, include_tables) + _geodiff_apply_changeset( + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables + ) + _geodiff_apply_changeset( + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.modified, tmp_base2their, ignored_tables, include_tables + ) else: logging.debug("Applying new version [WITH rebase]") tmp_conflicts = os.path.join(tmp_dir, f"{project_name}-dbsync-pull-conflicts") @@ -811,7 +815,9 @@ def pull(conn_cfg, mc): ignored_tables, include_tables, ) - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables) + _geodiff_apply_changeset( + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_base2their, ignored_tables, include_tables + ) os.remove(gpkg_basefile_old) conn = psycopg2.connect(conn_cfg.conn_info) @@ -1034,7 +1040,9 @@ def push(conn_cfg, mc): # update base schema in the DB logging.debug("Updating DB base schema...") - _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables, include_tables) + _geodiff_apply_changeset( + conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables, include_tables + ) _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version) From d58a2d951c05344a2f6d010e949e0337b4a8a4e7 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 10:44:58 +0200 Subject: [PATCH 12/15] add manual dispatch to specify branch for geodiff and mergin-py-client - defaults to master geodiff and install of client from PyPi branch is not specified --- .github/workflows/tests_mergin_db_sync.yaml | 38 ++++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 855b547..42ed37e 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -3,12 +3,22 @@ name: Tests for Mergin DB Sync on: push: paths: - - "test/**" - - "**.py" - - "requirements.txt" - - "requirements-dev.txt" - - "pyproject.toml" - - ".github/workflows/tests_mergin_db_sync.yaml" + - "test/**" + - "**.py" + - "requirements.txt" + - "requirements-dev.txt" + - "pyproject.toml" + - ".github/workflows/tests_mergin_db_sync.yaml" + workflow_dispatch: + inputs: + geodiff_branch: + description: "Geodiff branch to compile (default: master)" + required: false + default: "master" + mergin_client_branch: + description: "Mergin Python client branch from GitHub (leave empty to install from pip)" + required: false + default: "" env: TEST_GEODIFF_EXE: geodiff @@ -19,9 +29,7 @@ env: TEST_API_WORKSPACE: test-db-sync jobs: - Tests-for-Mergin-DB-Sync: - runs-on: ubuntu-latest services: @@ -36,20 +44,19 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Install Geodiff run: | sudo apt-get install libsqlite3-dev libpq-dev - git clone https://github.com/MerginMaps/geodiff.git + git clone --branch ${{ inputs.geodiff_branch || 'master' }} https://github.com/MerginMaps/geodiff.git cd geodiff mkdir build && cd build cmake -DWITH_POSTGRESQL=TRUE ../geodiff sudo make install - sudo cp geodiff /usr/local/bin + sudo cp geodiff /usr/local/bin - - name: Check Geodiff version + - name: Check Geodiff version run: geodiff version - + - name: Checkout uses: actions/checkout@v4 @@ -59,6 +66,11 @@ jobs: python3 -m pip install -r requirements.txt python3 -m pip install -r requirements-dev.txt + - name: Install Mergin client from GitHub + if: ${{ inputs.mergin_client_branch != '' }} + run: | + python3 -m pip install git+https://github.com/MerginMaps/python-api-client.git@${{ inputs.mergin_client_branch }} + - name: Run tests run: | pytest test --cov=. --cov-report=term-missing:skip-covered -vv From 7ee9c3716649a0490ab875a81ecf915712f90f73 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 11:22:13 +0200 Subject: [PATCH 13/15] if building from geodiff from branch we can also need custom build pygeodiff --- .github/workflows/tests_mergin_db_sync.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 42ed37e..d5be8d9 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -60,6 +60,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Install pygeodiff from source + if: ${{ inputs.mergin_client_branch != '' }} + run: | + python3 -m pip install scikit-build cmake ninja + cd geodiff + python3 -m pip install . + - name: Install Python dependencies run: | python3 -m pip install --upgrade pip From 7dea08fe508b7f887d863ead1b42d224d6cb6ae5 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 11:22:38 +0200 Subject: [PATCH 14/15] do not update dependencies here it can override custom pygeodiff --- .github/workflows/tests_mergin_db_sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index d5be8d9..6cb14f2 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -76,7 +76,7 @@ jobs: - name: Install Mergin client from GitHub if: ${{ inputs.mergin_client_branch != '' }} run: | - python3 -m pip install git+https://github.com/MerginMaps/python-api-client.git@${{ inputs.mergin_client_branch }} + python3 -m pip install --no-deps git+https://github.com/MerginMaps/python-api-client.git@${{ inputs.mergin_client_branch }} - name: Run tests run: | From 3dda7756d1df7def33ca1e939c554a2adc97e572 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 26 Jun 2026 13:49:33 +0200 Subject: [PATCH 15/15] revert changes --- .github/workflows/tests_mergin_db_sync.yaml | 45 ++++++--------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 6cb14f2..855b547 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -3,22 +3,12 @@ name: Tests for Mergin DB Sync on: push: paths: - - "test/**" - - "**.py" - - "requirements.txt" - - "requirements-dev.txt" - - "pyproject.toml" - - ".github/workflows/tests_mergin_db_sync.yaml" - workflow_dispatch: - inputs: - geodiff_branch: - description: "Geodiff branch to compile (default: master)" - required: false - default: "master" - mergin_client_branch: - description: "Mergin Python client branch from GitHub (leave empty to install from pip)" - required: false - default: "" + - "test/**" + - "**.py" + - "requirements.txt" + - "requirements-dev.txt" + - "pyproject.toml" + - ".github/workflows/tests_mergin_db_sync.yaml" env: TEST_GEODIFF_EXE: geodiff @@ -29,7 +19,9 @@ env: TEST_API_WORKSPACE: test-db-sync jobs: + Tests-for-Mergin-DB-Sync: + runs-on: ubuntu-latest services: @@ -44,40 +36,29 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: + - name: Install Geodiff run: | sudo apt-get install libsqlite3-dev libpq-dev - git clone --branch ${{ inputs.geodiff_branch || 'master' }} https://github.com/MerginMaps/geodiff.git + git clone https://github.com/MerginMaps/geodiff.git cd geodiff mkdir build && cd build cmake -DWITH_POSTGRESQL=TRUE ../geodiff sudo make install - sudo cp geodiff /usr/local/bin + sudo cp geodiff /usr/local/bin - - name: Check Geodiff version + - name: Check Geodiff version run: geodiff version - + - name: Checkout uses: actions/checkout@v4 - - name: Install pygeodiff from source - if: ${{ inputs.mergin_client_branch != '' }} - run: | - python3 -m pip install scikit-build cmake ninja - cd geodiff - python3 -m pip install . - - name: Install Python dependencies run: | python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt python3 -m pip install -r requirements-dev.txt - - name: Install Mergin client from GitHub - if: ${{ inputs.mergin_client_branch != '' }} - run: | - python3 -m pip install --no-deps git+https://github.com/MerginMaps/python-api-client.git@${{ inputs.mergin_client_branch }} - - name: Run tests run: | pytest test --cov=. --cov-report=term-missing:skip-covered -vv