From 7c1f8a63d59f1d246eed21e4f5cb5f8089ec4025 Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Thu, 18 Jun 2026 19:24:29 +0700 Subject: [PATCH 1/4] Add auto import for users and teams for ImportContest --- cmscontrib/ImportContest.py | 65 ++++-- cmscontrib/loaders/base_loader.py | 20 ++ cmscontrib/loaders/italy_yaml.py | 6 + cmscontrib/loaders/polygon.py | 4 + .../cmscontrib/ImportContestTest.py | 187 +++++++++++++++++- 5 files changed, 261 insertions(+), 21 deletions(-) diff --git a/cmscontrib/ImportContest.py b/cmscontrib/ImportContest.py index c81d9b9762..f1c03ba9bc 100755 --- a/cmscontrib/ImportContest.py +++ b/cmscontrib/ImportContest.py @@ -10,6 +10,7 @@ # Copyright © 2021 Manuel Gundlach # Copyright © 2026 Tobias Lenz # Copyright © 2026 Chuyang Wang +# Copyright © 2026 Pasit Sangprachathanarak # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -50,8 +51,10 @@ from cms.db.filecacher import FileCacher from cmscontrib.importing import ImportDataError, update_contest, \ update_group, update_task +from cmscontrib.ImportTeam import TeamImporter +from cmscontrib.ImportUser import UserImporter from cmscontrib.loaders import choose_loader, build_epilog -from cmscontrib.loaders.base_loader import BaseLoader, ContestLoader +from cmscontrib.loaders.base_loader import ContestLoader logger = logging.getLogger(__name__) @@ -74,6 +77,8 @@ def __init__( update_tasks: bool, no_statements: bool, delete_stale_participations: bool, + auto_import_users: bool, + auto_import_teams: bool, loader_class: type[ContestLoader], ): self.yes = yes @@ -83,6 +88,8 @@ def __init__( self.update_tasks = update_tasks self.no_statements = no_statements self.delete_stale_participations = delete_stale_participations + self.auto_import_users = auto_import_users + self.auto_import_teams = auto_import_teams self.file_cacher = FileCacher() self.loader = loader_class(os.path.abspath(path), self.file_cacher) @@ -265,9 +272,8 @@ def _task_to_db( task.contest = contest return task - @staticmethod def _participation_to_db( - session: Session, contest: Contest, new_p: dict + self, session: Session, contest: Contest, new_p: dict ) -> Participation: """Add the new participation to the DB and attach it to the contest @@ -287,19 +293,42 @@ def _participation_to_db( session.query(User).filter(User.username == new_p["username"]).first() ) if user is None: - # FIXME: it would be nice to automatically try to import. - raise ImportDataError("User \"%s\" not found in database. " - "Use cmsImportUser to import it." % - new_p["username"]) + if not self.auto_import_users: + raise ImportDataError("User \"%s\" not found in database. " + "Use cmsImportUser to import it." % + new_p["username"]) + user_loader = self.loader.get_user_loader(new_p["username"]) + if user_loader is None: + raise ImportDataError( + "User \"%s\" not found in database. " + "Use cmsImportUser to import it." % + new_p["username"]) + user = user_loader.get_user() + if user is None: + raise ImportDataError( + "Could not import user \"%s\"." % + new_p["username"]) + user = UserImporter._user_to_db(session, user) team: Team | None = ( session.query(Team).filter(Team.code == new_p.get("team")).first() ) if team is None and new_p.get("team") is not None: - # FIXME: it would be nice to automatically try to import. - raise ImportDataError("Team \"%s\" not found in database. " - "Use cmsImportTeam to import it." - % new_p.get("team")) + if not self.auto_import_teams: + raise ImportDataError("Team \"%s\" not found in database. " + "Use cmsImportTeam to import it." + % new_p.get("team")) + team_loader = self.loader.get_team_loader(new_p.get("team")) + if team_loader is None: + raise ImportDataError( + "Team \"%s\" not found in database. " + "Use cmsImportTeam to import it." + % new_p.get("team")) + team = team_loader.get_team() + if team is None: + raise ImportDataError( + "Could not import team \"%s\"." % new_p.get("team")) + team = TeamImporter._team_to_db(session, team) # Check that the participation is not already defined. p: Participation | None = ( @@ -464,6 +493,16 @@ def main(): action="store_true", help="do not import / update task statements" ) + parser.add_argument( + "--no-auto-import-users", + action="store_true", + help="do not automatically import missing users" + ) + parser.add_argument( + "--no-auto-import-teams", + action="store_true", + help="do not automatically import missing teams" + ) parser.add_argument( "--delete-stale-participations", action="store_true", @@ -493,6 +532,10 @@ def main(): update_tasks=args.update_tasks, no_statements=args.no_statements, delete_stale_participations=args.delete_stale_participations, + auto_import_users=not ( + args.no_auto_import or args.no_auto_import_users), + auto_import_teams=not ( + args.no_auto_import or args.no_auto_import_teams), loader_class=loader_class) success = importer.do_import() return 0 if success is True else 1 diff --git a/cmscontrib/loaders/base_loader.py b/cmscontrib/loaders/base_loader.py index fa28a798f5..36aa3c98a1 100644 --- a/cmscontrib/loaders/base_loader.py +++ b/cmscontrib/loaders/base_loader.py @@ -335,3 +335,23 @@ def get_task_loader(self, taskname: str) -> TaskLoader: """ pass + + def get_user_loader(self, username: str) -> UserLoader | None: + """Return a loader class for the user with the given username. + + username: username of the user. + + return: loader for the user with username. + + """ + return None + + def get_team_loader(self, teamcode: str) -> TeamLoader | None: + """Return a loader class for the team with the given code. + + teamcode: code of the team. + + return: loader for the team with code teamcode. + + """ + return None diff --git a/cmscontrib/loaders/italy_yaml.py b/cmscontrib/loaders/italy_yaml.py index 760a34960a..16c510c3a0 100644 --- a/cmscontrib/loaders/italy_yaml.py +++ b/cmscontrib/loaders/italy_yaml.py @@ -162,6 +162,12 @@ def detect(path): def get_task_loader(self, taskname): return YamlLoader(os.path.join(self.path, taskname), self.file_cacher) + def get_user_loader(self, username): + return YamlLoader(os.path.join(self.path, username), self.file_cacher) + + def get_team_loader(self, teamcode): + return YamlLoader(os.path.join(self.path, teamcode), self.file_cacher) + def get_contest(self): """See docstring in class ContestLoader.""" if not os.path.exists(os.path.join(self.path, "contest.yaml")): diff --git a/cmscontrib/loaders/polygon.py b/cmscontrib/loaders/polygon.py index 29030fc994..9c0db6d367 100644 --- a/cmscontrib/loaders/polygon.py +++ b/cmscontrib/loaders/polygon.py @@ -358,6 +358,10 @@ def get_task_loader(self, taskname): taskpath = os.path.join(self.path, "problems", taskname) return PolygonTaskLoader(taskpath, self.file_cacher) + def get_user_loader(self, username): + userpath = os.path.join(self.path, username) + return PolygonUserLoader(userpath, self.file_cacher) + def get_contest(self): """See docstring in class Loader. diff --git a/cmstestsuite/unit_tests/cmscontrib/ImportContestTest.py b/cmstestsuite/unit_tests/cmscontrib/ImportContestTest.py index 045f745f01..514476ed5d 100755 --- a/cmstestsuite/unit_tests/cmscontrib/ImportContestTest.py +++ b/cmstestsuite/unit_tests/cmscontrib/ImportContestTest.py @@ -22,35 +22,51 @@ from cmstestsuite.unit_tests.databasemixin import DatabaseMixin -from cms.db import Contest, SessionGen, Submission, User +from cms.db import Contest, SessionGen, Submission, Team, User from cmscontrib.ImportContest import ContestImporter -from cmscontrib.loaders.base_loader import ContestLoader, TaskLoader +from cmscontrib.loaders.base_loader import ( + ContestLoader, + TaskLoader, + TeamLoader, + UserLoader, +) def fake_loader_factory( contest: Contest, contest_has_changed: bool = False, tasks: list[tuple[str, bool]] | None = None, - usernames: list[str] | None = None, + participations: list[dict] | None = None, + users: list[User] | None = None, + teams: list[Team] | None = None, ): """Return a Loader class always returning the same information contest: the contest to return contest_has_changed: what to return from contest_has_changed tasks: list of task names and whether they have changed - usernames: list of usernames of participations + participations: list of participations + users: list of importable users + teams: list of importable teams """ tasks = tasks if tasks is not None else [] - usernames = usernames if usernames is not None else [] + participations = participations if participations is not None else [] + users = users if users is not None else [] + teams = teams if teams is not None else [] task_name_list = [t.name for t, has_changed in tasks] tasks_by_name = dict((t.name, { "task": t, "has_changed": has_changed }) for t, has_changed in tasks) - participations = [{"username": u} for u in usernames] + users_by_name = ( + users if isinstance(users, dict) else dict((u.username, u) for u in users) + ) + teams_by_code = ( + teams if isinstance(teams, dict) else dict((t.code, t) for t in teams) + ) class FakeLoader(ContestLoader): @staticmethod @@ -78,6 +94,36 @@ def task_has_changed(self): return FakeTaskLoader(self.path, self.file_cacher) + def get_user_loader(self, username): + + class FakeUserLoader(UserLoader): + @staticmethod + def detect(path): + return True + + def get_user(self): + return users_by_name.get(username, None) + + def user_has_changed(self): + return True + + return FakeUserLoader(self.path, self.file_cacher) + + def get_team_loader(self, teamcode): + + class FakeTeamLoader(TeamLoader): + @staticmethod + def detect(path): + return True + + def get_team(self): + return teams_by_code.get(teamcode, None) + + def team_has_changed(self): + return True + + return FakeTeamLoader(self.path, self.file_cacher) + return FakeLoader @@ -115,13 +161,21 @@ def tearDown(self): def do_import(contest, tasks, participations, contest_has_changed=False, update_contest=False, import_tasks=False, update_tasks=False, - delete_stale_participations=False): + delete_stale_participations=False, + auto_import_users=True, auto_import_teams=True, + users=None, teams=None): """Create an importer and call do_import in a convenient way""" + participations = [ + p if isinstance(p, dict) else {"username": p} + for p in participations + ] return ContestImporter( "path", True, False, import_tasks, update_contest, update_tasks, False, delete_stale_participations, + auto_import_users, auto_import_teams, fake_loader_factory(contest, contest_has_changed, - tasks, participations)).do_import() + tasks, participations, users, teams) + ).do_import() def assertContestInDb(self, name, description, task_names_and_titles, usernames_and_last_names): @@ -151,6 +205,15 @@ def assertSubmissionCount(self, count): with SessionGen() as session: self.assertEqual(session.query(Submission).count(), count) + def assertTeamInDb(self, code, name): + """Assert that the team with the given data is in the DB""" + with SessionGen() as session: + db_teams = session.query(Team).filter(Team.code == code).all() + self.assertEqual(len(db_teams), 1) + team = db_teams[0] + self.assertEqual(team.code, code) + self.assertEqual(team.name, name) + def test_import_task_in_db_not_attached(self): # Completely new contest, the task is already in the DB, not attached # to any contest. The import should succeed and the task should made @@ -295,9 +358,38 @@ def test_import_participation_in_db(self): [(self.username, self.last_name)]) self.assertSubmissionCount(1) - def test_import_participation_not_in_db(self): + def test_import_participation_not_in_db_imported(self): + # Completely new contest, no tasks, a new participation whose user is + # not in the DB but can be imported by the loader. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + username = "new_username" + last_name = "new_last_name" + user = self.get_user(username=username, last_name=last_name) + ret = self.do_import(contest, [], [username], users=[user]) + + self.assertTrue(ret) + self.assertContestInDb(name, description, [], [(username, last_name)]) + self.assertSubmissionCount(1) + + def test_import_participation_not_in_db_fail_without_user_auto_import(self): # Completely new contest, no tasks, a new participation but the user - # is not in the DB, so it should fail. + # is not in the DB and auto import is disabled, so it should fail. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + username = "new_username" + user = self.get_user(username=username) + ret = self.do_import( + contest, [], [username], auto_import_users=False, users=[user]) + + self.assertFalse(ret) + self.assertSubmissionCount(1) + + def test_import_participation_not_in_db_fail_if_user_loader_missing(self): + # Completely new contest, no tasks, a new participation whose user is + # neither in the DB nor available through the loader, so it should fail. name = "new_name" description = "new_desc" contest = self.get_contest(name=name, description=description) @@ -307,6 +399,81 @@ def test_import_participation_not_in_db(self): self.assertFalse(ret) self.assertSubmissionCount(1) + def test_import_participation_not_in_db_fail_if_user_loader_duplicates(self): + # If the loader produces a user that already exists under a different + # referenced username, treat it like cmsImportUser and fail. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + user = self.get_user(username=self.username) + ret = self.do_import( + contest, [], ["new_username"], users={"new_username": user}) + + self.assertFalse(ret) + self.assertSubmissionCount(1) + + def test_import_participation_team_not_in_db_imported(self): + # Completely new contest, no tasks, a new participation whose team is + # not in the DB but can be imported by the loader. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + team_code = "new_team_code" + team_name = "new_team_name" + team = self.get_team(code=team_code, name=team_name) + ret = self.do_import( + contest, [], [{"username": self.username, "team": team_code}], + teams=[team]) + + self.assertTrue(ret) + self.assertContestInDb(name, description, [], + [(self.username, self.last_name)]) + self.assertTeamInDb(team_code, team_name) + self.assertSubmissionCount(1) + + def test_import_participation_team_not_in_db_fail_without_team_auto_import(self): + # Completely new contest, no tasks, a new participation whose team is + # not in the DB and auto import is disabled, so it should fail. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + team_code = "new_team_code" + team = self.get_team(code=team_code) + ret = self.do_import( + contest, [], [{"username": self.username, "team": team_code}], + auto_import_teams=False, teams=[team]) + + self.assertFalse(ret) + self.assertSubmissionCount(1) + + def test_import_participation_team_not_in_db_fail_if_team_loader_missing(self): + # Completely new contest, no tasks, a new participation whose team is + # neither in the DB nor available through the loader, so it should fail. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + team_code = "new_team_code" + ret = self.do_import( + contest, [], [{"username": self.username, "team": team_code}]) + + self.assertFalse(ret) + self.assertSubmissionCount(1) + + def test_import_participation_team_not_in_db_fail_if_team_loader_duplicates(self): + # If the loader produces a team that already exists under a different + # referenced code, treat it like cmsImportTeam and fail. + name = "new_name" + description = "new_desc" + contest = self.get_contest(name=name, description=description) + team = self.get_team(code=self.add_team().code) + self.session.commit() + ret = self.do_import( + contest, [], [{"username": self.username, "team": "new_team_code"}], + teams={"new_team_code": team}) + + self.assertFalse(ret) + self.assertSubmissionCount(1) + def test_delete_stale_participations(self): # Update the existing contest, task not updated, we also ask to # delete participations that we do not pass. From d60803511140cdf4d07daa1f288cd4ead0e48f26 Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Thu, 18 Jun 2026 20:57:34 +0700 Subject: [PATCH 2/4] Update ImportContest.py --- cmscontrib/ImportContest.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cmscontrib/ImportContest.py b/cmscontrib/ImportContest.py index f1c03ba9bc..eb1a62f68a 100755 --- a/cmscontrib/ImportContest.py +++ b/cmscontrib/ImportContest.py @@ -77,8 +77,8 @@ def __init__( update_tasks: bool, no_statements: bool, delete_stale_participations: bool, - auto_import_users: bool, - auto_import_teams: bool, + no_auto_import_users: bool, + no_auto_import_teams: bool, loader_class: type[ContestLoader], ): self.yes = yes @@ -88,8 +88,8 @@ def __init__( self.update_tasks = update_tasks self.no_statements = no_statements self.delete_stale_participations = delete_stale_participations - self.auto_import_users = auto_import_users - self.auto_import_teams = auto_import_teams + self.no_auto_import_users = no_auto_import_users + self.no_auto_import_teams = no_auto_import_teams self.file_cacher = FileCacher() self.loader = loader_class(os.path.abspath(path), self.file_cacher) @@ -293,7 +293,7 @@ def _participation_to_db( session.query(User).filter(User.username == new_p["username"]).first() ) if user is None: - if not self.auto_import_users: + if self.no_auto_import_users: raise ImportDataError("User \"%s\" not found in database. " "Use cmsImportUser to import it." % new_p["username"]) @@ -314,7 +314,7 @@ def _participation_to_db( session.query(Team).filter(Team.code == new_p.get("team")).first() ) if team is None and new_p.get("team") is not None: - if not self.auto_import_teams: + if self.no_auto_import_teams: raise ImportDataError("Team \"%s\" not found in database. " "Use cmsImportTeam to import it." % new_p.get("team")) @@ -532,10 +532,8 @@ def main(): update_tasks=args.update_tasks, no_statements=args.no_statements, delete_stale_participations=args.delete_stale_participations, - auto_import_users=not ( - args.no_auto_import or args.no_auto_import_users), - auto_import_teams=not ( - args.no_auto_import or args.no_auto_import_teams), + no_auto_import_users=args.no_auto_import_users, + no_auto_import_teams=args.no_auto_import_teams, loader_class=loader_class) success = importer.do_import() return 0 if success is True else 1 From ffc6f2b2857b3ea8697a69ede14814bac492e246 Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Thu, 18 Jun 2026 21:03:47 +0700 Subject: [PATCH 3/4] Update ImportContest.py --- cmscontrib/ImportContest.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmscontrib/ImportContest.py b/cmscontrib/ImportContest.py index eb1a62f68a..d9774719a1 100755 --- a/cmscontrib/ImportContest.py +++ b/cmscontrib/ImportContest.py @@ -77,8 +77,8 @@ def __init__( update_tasks: bool, no_statements: bool, delete_stale_participations: bool, - no_auto_import_users: bool, - no_auto_import_teams: bool, + auto_import_users: bool, + auto_import_teams: bool, loader_class: type[ContestLoader], ): self.yes = yes @@ -88,8 +88,8 @@ def __init__( self.update_tasks = update_tasks self.no_statements = no_statements self.delete_stale_participations = delete_stale_participations - self.no_auto_import_users = no_auto_import_users - self.no_auto_import_teams = no_auto_import_teams + self.auto_import_users = auto_import_users + self.auto_import_teams = auto_import_teams self.file_cacher = FileCacher() self.loader = loader_class(os.path.abspath(path), self.file_cacher) @@ -293,7 +293,7 @@ def _participation_to_db( session.query(User).filter(User.username == new_p["username"]).first() ) if user is None: - if self.no_auto_import_users: + if not self.auto_import_users: raise ImportDataError("User \"%s\" not found in database. " "Use cmsImportUser to import it." % new_p["username"]) @@ -314,7 +314,7 @@ def _participation_to_db( session.query(Team).filter(Team.code == new_p.get("team")).first() ) if team is None and new_p.get("team") is not None: - if self.no_auto_import_teams: + if not self.auto_import_teams: raise ImportDataError("Team \"%s\" not found in database. " "Use cmsImportTeam to import it." % new_p.get("team")) @@ -494,14 +494,14 @@ def main(): help="do not import / update task statements" ) parser.add_argument( - "--no-auto-import-users", + "-iu", "--import-users", action="store_true", - help="do not automatically import missing users" + help="import users if they do not exist" ) parser.add_argument( - "--no-auto-import-teams", + "-it", "--import-teams", action="store_true", - help="do not automatically import missing teams" + help="import teams if they do not exist" ) parser.add_argument( "--delete-stale-participations", @@ -532,8 +532,8 @@ def main(): update_tasks=args.update_tasks, no_statements=args.no_statements, delete_stale_participations=args.delete_stale_participations, - no_auto_import_users=args.no_auto_import_users, - no_auto_import_teams=args.no_auto_import_teams, + auto_import_users=args.auto_import_users, + auto_import_teams=args.auto_import_teams, loader_class=loader_class) success = importer.do_import() return 0 if success is True else 1 From 999fac704dd83c86a07d9604d5fe4a6fc922d9d8 Mon Sep 17 00:00:00 2001 From: Pasit Sangprachathanarak Date: Thu, 18 Jun 2026 21:05:46 +0700 Subject: [PATCH 4/4] Update ImportContest.py --- cmscontrib/ImportContest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmscontrib/ImportContest.py b/cmscontrib/ImportContest.py index d9774719a1..ce4bf442fe 100755 --- a/cmscontrib/ImportContest.py +++ b/cmscontrib/ImportContest.py @@ -532,8 +532,8 @@ def main(): update_tasks=args.update_tasks, no_statements=args.no_statements, delete_stale_participations=args.delete_stale_participations, - auto_import_users=args.auto_import_users, - auto_import_teams=args.auto_import_teams, + auto_import_users=args.import_users, + auto_import_teams=args.import_teams, loader_class=loader_class) success = importer.do_import() return 0 if success is True else 1