diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 3f9f3b3dbc..78c4fc975d 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -284,6 +284,19 @@ db.url=jdbc:h2:./lift_proto.db;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE #db.driver=com.microsoft.sqlserver.jdbc.SQLServerDriver #db.url=jdbc:sqlserver://localhost:1433;databaseName=PSD2_OBP;user=OBPApi;password=********; +## Dynamic Entity list-query backend. +## Controls how GET-all reads on Dynamic Entities are served: +## inmemory (default) = portable in-memory filter/sort/paginate; works on any database. +## Cross-entity joins (obp_exists / obp_not_exists) are NOT available and return 400. +## auto = push filtering/sort/paginate AND obp_exists/obp_not_exists joins to the database +## via per-entity projection tables + indexes. Enables joins. +## This is a strategy toggle, not a vendor name: with "auto" the projection backend matching your db.url +## vendor is selected automatically (Postgres now; SQL Server when that backend ships) - the vendor is +## detected from db.url, so there is no per-database value to set here. "auto" requires a Postgres (or +## SQL Server) db.url; on any other vendor it falls back to inmemory. Projection tables are provisioned +## when a Dynamic Entity definition is created/updated, so re-save existing definitions after enabling. +#dynamic_entity.indexing.backend=inmemory + ## Enable remote Akka actor for data split ## If set to true, must set hostname and port ## of remote machine diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 681f0ad0ee..9897172202 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -29,6 +29,7 @@ package bootstrap.liftweb import org.json4s._ import code.CustomerDependants.MappedCustomerDependant import code.DynamicData.DynamicData +import code.DynamicData.DynamicDataAccess import code.DynamicEndpoint.DynamicEndpoint import code.UserRefreshes.MappedUserRefreshes import code.abacrule.AbacRule @@ -951,6 +952,7 @@ object ToSchemify extends MdcLoggable { WebUiProps, DynamicEntity, DynamicData, + DynamicDataAccess, code.api.dynamic.entity.projection.DynamicEntityIndex, DynamicEndpoint, AccountIdMapping, diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala index 82f248cdeb..0fb8a25307 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala @@ -27,10 +27,10 @@ package code.api.dynamic.entity import cats.data.{Kleisli, OptionT} import cats.effect.IO -import code.DynamicData.{DynamicData, DynamicDataProvider} +import code.DynamicData.{DynamicData, DynamicDataProvider, DynamicDataAccessProvider, DynamicDataAccessPermission} import code.api.Constant.PARAM_LOCALE -import code.api.dynamic.entity.helper.{CommunityEntityName, DynamicEntityHelper, DynamicEntityInfo, EntityName, PublicEntityName} -import code.api.dynamic.entity.query.{FieldSpec, InMemoryQueryExecutor, QueryParamParser, QueryPlan, QueryPlanner} +import code.api.dynamic.entity.helper.{CommunityEntityName, DynamicEntityHelper, DynamicEntityInfo, EntityAccessName, EntityName, PublicEntityName} +import code.api.dynamic.entity.query.{FieldSpec, InMemoryQueryExecutor, JoinTargetInfo, QueryParamParser, QueryPlan, QueryPlanner} import code.api.dynamic.entity.projection.{IndexingCapabilities, PostgresProjectionBackend, ProjectionProvisioner} import cats.effect.unsafe.implicits.{global => ioRuntime} // aliased: avoids clashing with the EC `global` imported below import code.api.util.APIUtil._ @@ -111,12 +111,19 @@ object Http4sDynamicEntity extends MdcLoggable { private def deIndexedFields(bankId: Option[String], entityName: String): Map[String, FieldSpec] = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).map(_.indexedFields).getOrElse(Map.empty) + private def deReferenceFields(bankId: Option[String], entityName: String): Map[String, String] = + DynamicEntityHelper.definitionsMap.get((bankId, entityName)).map(_.referenceFields).getOrElse(Map.empty) + + /** Resolve a join-target (child) entity's indexed + reference fields for the planner (same bank scope). */ + private def childJoinInfo(bankId: Option[String])(child: String): Option[JoinTargetInfo] = + DynamicEntityHelper.definitionsMap.get((bankId, child)).map(i => JoinTargetInfo(i.indexedFields, i.referenceFields)) + /** Parse + validate list-read query params into a QueryPlan; fail 400 (clear message) on any error. */ private def buildQueryPlan(req: Request[IO], bankId: Option[String], entityName: String, cc: Option[CallContext]): Future[QueryPlan] = { - val planned = for { - parsed <- QueryParamParser.parse(queryParams(req)) - plan <- QueryPlanner.plan(parsed._1, parsed._2, parsed._3, deIndexedFields(bankId, entityName)) - } yield plan + val planned = QueryParamParser.parse(queryParams(req)).flatMap { case (filters, joins, sort, page) => + QueryPlanner.plan(filters, joins, sort, page, entityName, + deIndexedFields(bankId, entityName), deReferenceFields(bankId, entityName), childJoinInfo(bankId)) + } planned match { case Right(plan) => Future.successful(plan) case Left(err) => Helper.booleanToFuture(err.message, 400, cc = cc) { false }.map(_ => QueryPlan.empty) @@ -278,15 +285,34 @@ object Http4sDynamicEntity extends MdcLoggable { private case object UseProjection extends ProjDecision private case object PendingProjection extends ProjDecision private case object UseInMemory extends ProjDecision + private case object JoinsNeedProjection extends ProjDecision // joins present but projection unavailable -> 400 private def legacyParamsPresent(req: Request[IO]): Boolean = queryParams(req).keys.exists(k => k != PARAM_LOCALE && !k.startsWith("obp_")) + /** True if the request carries any obp_exists / obp_not_exists join clause (used to reject joins on + * the non-projection get-all paths: public, community, row-level). */ + private def joinParamsPresent(req: Request[IO]): Boolean = + queryParams(req).keys.exists(k => k.startsWith("obp_exists[") || k.startsWith("obp_not_exists[")) + private def planFields(plan: QueryPlan): List[String] = (plan.filters.map(_.field) ++ plan.sort.map(_.field)).distinct private def decideProjection(req: Request[IO], bankId: Option[String], entityName: String, plan: QueryPlan): ProjDecision = - if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req) || planFields(plan).isEmpty) UseInMemory + if (plan.joins.nonEmpty) { + // Joins are projection-only. Legacy bare params can't combine with joins (they force in-memory). + if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req)) JoinsNeedProjection + else { + val parentReady = ProjectionProvisioner.readyFields(bankId, entityName) + val parentFieldsReady = planFields(plan).forall(parentReady.contains) + val joinsReady = plan.joins.forall { j => + val childReady = ProjectionProvisioner.readyFields(bankId, j.childEntity) + val linkReady = if (j.onChild) childReady.contains(j.linkField) else parentReady.contains(j.linkField) + linkReady && j.predicate.map(_.field).forall(childReady.contains) + } + if (parentFieldsReady && joinsReady) UseProjection else PendingProjection + } + } else if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req) || planFields(plan).isEmpty) UseInMemory else { val ready = ProjectionProvisioner.readyFields(bankId, entityName) if (planFields(plan).forall(ready.contains)) UseProjection else PendingProjection @@ -295,10 +321,201 @@ object Http4sDynamicEntity extends MdcLoggable { private def projectionList(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean, plan: QueryPlan): Future[JArray] = PostgresProjectionBackend.query(entityName, bankId, userId, isPersonalEntity, plan).map(JArray(_)).unsafeToFuture()(ioRuntime) + // ----- row-level access (use_row_level_access) ----- + // When an entity opts into row-level access the per-entity GET/UPDATE/DELETE roles do NOT + // gate access; the DynamicDataAccess ACL decides per row. Rows are fetched UNSCOPED (community + // path) so ownership doesn't pre-filter, then the ACL gates. Row-level entities are local by + // definition (§8.5 guard), so calling the local provider directly violates no abstraction. + // See ideas/DYNAMIC_ENTITY_ROW_LEVEL_ACCESS.md §5. + + private def aclVend = DynamicDataAccessProvider.provider.vend + private def dataVend = DynamicDataProvider.connectorMethodProvider.vend + private def isRowLevel(bankId: Option[String], entityName: String): Boolean = + DynamicEntityHelper.definitionsMap.get((bankId, entityName)).exists(_.useRowLevelAccess) + + private def rowLevelGet(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String): Future[JValue] = { + val isGetAll = StringUtils.isBlank(id) + val operation: DynamicEntityOperation = if (isGetAll) GET_ALL else GET_ONE + val callContext0 = enrichCallContext(cc, operation, entityName, bankId, "") + val operationId = callContext0.operationId.orNull + for { + _ <- failIf(beforeIntercept(callContext0, operationId), Some(callContext0)) + (Full(u), callContext) <- authenticatedAccess(callContext0) + (_, callContext) <- bankCheck(bankId, callContext) + // Row-level: type-level get role is NOT required — the ACL decides per row. + _ <- failIf(afterIntercept(callContext, operationId), callContext) + // Row-level get-all is served in-memory (ACL-gated); joins require the projection backend. + _ <- if (isGetAll && joinParamsPresent(req)) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false } + else Future.successful(true) + result <- if (isGetAll) Future { + // In-memory floor: fetch all rows (unscoped) and keep those the ACL marks readable. + // (The projection EXISTS backend for row-level get-all is a documented follow-up; the + // in-memory path is always correct, just not index-accelerated.) + val readable = aclVend.getReadableRowIds(u.userId, entityName, bankId).toSet + val readableRows = dataVend.getAllCommunity(bankId, entityName).filter(_.dynamicDataId.exists(readable.contains)) + val readableJson: JArray = JArray(readableRows.map(r => parse(r.dataJson))) + val filtered = filterDynamicObjects(readableJson, queryParams(req)) + wrapBankId(bankId, (listName(entityName) -> applyReadRestrictions(filtered, bankId, entityName, Some(u.userId)))) + } else { + val box: Box[JValue] = dataVend.getCommunity(bankId, entityName, id).map(it => parse(it.dataJson)) + for { + // Hide existence: a row you cannot read is indistinguishable from a missing row (404). + _ <- Helper.booleanToFuture(notFoundMsg(entityName, id, bankId), 404, cc = callContext) { + box.isDefined && aclVend.allows(id, u.userId, DynamicDataAccessPermission.Read) + } + } yield { + val singleObject: JValue = unboxResult(box, entityName) + wrapBankId(bankId, (singleName(entityName) -> applyReadRestrictions(singleObject, bankId, entityName, Some(u.userId)))) + } + } + } yield result + } + + private def rowLevelPut(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String): Future[JValue] = { + val callContext0 = enrichCallContext(cc, UPDATE, entityName, bankId, "") + val operationId = callContext0.operationId.orNull + for { + _ <- failIf(beforeIntercept(callContext0, operationId), Some(callContext0)) + (Full(u), callContext) <- authenticatedAccess(callContext0) + (_, callContext) <- bankCheck(bankId, callContext) + _ <- failIf(afterIntercept(callContext, operationId), callContext) + json <- NewStyle.function.tryons(InvalidJsonFormat, 400, callContext) { parse(cc.httpBody.getOrElse("")) } + existing: Box[JValue] = dataVend.getCommunity(bankId, entityName, id).map(it => parse(it.dataJson)) + _ <- Helper.booleanToFuture(notFoundMsg(entityName, id, bankId), 404, cc = callContext) { existing.isDefined } + _ <- Helper.booleanToFuture(s"$UserHasMissingRoles update access on this row", 403, cc = callContext) { + aclVend.allows(id, u.userId, DynamicDataAccessPermission.Update) } + // Field-level write roles still apply on top of the row ACL. + updateJson = preserveRestrictedOnPut(json.asInstanceOf[JObject], existing, writeRestrictedFieldsOf(bankId, entityName)) + box: Box[JValue] = dataVend.updateCommunity(bankId, entityName, updateJson, id).map(it => parse(it.dataJson)) + singleObject: JValue = unboxResult(box, entityName) + } yield wrapBankId(bankId, (singleName(entityName) -> singleObject)) + } + + private def rowLevelPatch(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String): Future[JValue] = { + val callContext0 = enrichCallContext(cc, UPDATE, entityName, bankId, "") + val operationId = callContext0.operationId.orNull + for { + _ <- failIf(beforeIntercept(callContext0, operationId), Some(callContext0)) + (Full(u), callContext) <- authenticatedAccess(callContext0) + (_, callContext) <- bankCheck(bankId, callContext) + _ <- failIf(afterIntercept(callContext, operationId), callContext) + json <- NewStyle.function.tryons(InvalidJsonFormat, 400, callContext) { parse(cc.httpBody.getOrElse("")) } + bodyObj = json.asInstanceOf[JObject] + // Row ACL replaces the entity-update role; per-field write roles still apply (requireEntityRole = false). + _ <- Helper.booleanToFuture(s"$UserHasMissingRoles update access on this row", 403, cc = callContext) { + aclVend.allows(id, u.userId, DynamicDataAccessPermission.Update) } + missingRoles = missingPatchRoleNames(bodyObj.obj.map(_.name), bankId, entityName, u.userId, requireEntityRole = false) + _ <- Helper.booleanToFuture(s"$UserHasMissingRoles ${missingRoles.mkString(", ")}", 403, cc = callContext) { missingRoles.isEmpty } + existing: Box[JValue] = dataVend.getCommunity(bankId, entityName, id).map(it => parse(it.dataJson)) + _ <- Helper.booleanToFuture(notFoundMsg(entityName, id, bankId), 404, cc = callContext) { existing.isDefined } + mergedJson = mergePatch(DynamicEntityHelper.definitionsMap.get((bankId, entityName)), existing, bodyObj) + box: Box[JValue] = dataVend.updateCommunity(bankId, entityName, mergedJson, id).map(it => parse(it.dataJson)) + singleObject: JValue = unboxResult(box, entityName) + } yield wrapBankId(bankId, (singleName(entityName) -> singleObject)) + } + + private def rowLevelDelete(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String): Future[JValue] = { + val callContext0 = enrichCallContext(cc, DELETE, entityName, bankId, "") + val operationId = callContext0.operationId.orNull + for { + _ <- failIf(beforeIntercept(callContext0, operationId), Some(callContext0)) + (Full(u), callContext) <- authenticatedAccess(callContext0) + (_, callContext) <- bankCheck(bankId, callContext) + _ <- failIf(afterIntercept(callContext, operationId), callContext) + existing: Box[JValue] = dataVend.getCommunity(bankId, entityName, id).map(it => parse(it.dataJson)) + _ <- Helper.booleanToFuture(notFoundMsg(entityName, id, bankId), 404, cc = callContext) { existing.isDefined } + _ <- Helper.booleanToFuture(s"$UserHasMissingRoles delete access on this row", 403, cc = callContext) { + aclVend.allows(id, u.userId, DynamicDataAccessPermission.Delete) } + _ = dataVend.deleteCommunity(bankId, entityName, id) + _ = aclVend.deleteAllForRow(id) // cascade the ACL rows for the deleted data row + } yield JObject(Nil) + } + + // ----- row-level access management: share / list / revoke (...///access) ----- + // §6. Authorisation: ACL CanGrant on the row OR the per-entity admin role. The owner holds + // CanGrant via their bootstrap row, so they share their own records with no role (§8.1). + + private def boolField(o: JObject, name: String, default: Boolean): Boolean = + (o \ name) match { case JBool(b) => b; case _ => default } + private def strField(o: JObject, name: String): Option[String] = + (o \ name) match { case JString(s) if s.nonEmpty => Some(s); case _ => None } + + private def rowAccessRowJson(a: code.DynamicData.DynamicDataAccessT): JObject = + ("user_id" -> a.userId) ~ + ("can_read" -> a.canRead) ~ + ("can_update" -> a.canUpdate) ~ + ("can_delete" -> a.canDelete) ~ + ("can_grant" -> a.canGrant) ~ + ("granted_by" -> a.grantedBy) + + private def rowAccessListJson(dataId: String): JObject = + ("access" -> JArray(aclVend.getAccessForRow(dataId).map(rowAccessRowJson))) + + // Shared preamble: before-intercept, auth, bank, flag-off (400), grant authorisation (403). + private def rowAccessAuthorise(cc: CallContext, bankId: Option[String], entityName: String, id: String, + operation: DynamicEntityOperation): Future[(User, Option[CallContext])] = { + val callContext0 = enrichCallContext(cc, operation, entityName, bankId, "") + val operationId = callContext0.operationId.orNull + for { + _ <- failIf(beforeIntercept(callContext0, operationId), Some(callContext0)) + (Full(u), callContext) <- authenticatedAccess(callContext0) + (_, callContext2) <- bankCheck(bankId, callContext) + _ <- Helper.booleanToFuture(RowLevelAccessNotEnabled, 400, cc = callContext2) { isRowLevel(bankId, entityName) } + _ <- Helper.booleanToFuture(s"$UserHasMissingRoles grant access on this row", 403, cc = callContext2) { + aclVend.allows(id, u.userId, DynamicDataAccessPermission.Grant) || + hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canGrantRowAccessRole(entityName, bankId)) + } + } yield (u, callContext2) + } + + private def listRowAccess(req: Request[IO], bankId: Option[String], entityName: String, id: String): IO[Response[IO]] = + EndpointHelpers.executeAndRespond(req) { cc => + for { + _ <- rowAccessAuthorise(cc, bankId, entityName, id, GET_ALL) + } yield rowAccessListJson(id) + } + + private def upsertRowAccess(req: Request[IO], bankId: Option[String], entityName: String, id: String): IO[Response[IO]] = + EndpointHelpers.executeAndRespond(req) { cc => + for { + (granter, callContext) <- rowAccessAuthorise(cc, bankId, entityName, id, UPDATE) + json <- NewStyle.function.tryons(InvalidJsonFormat, 400, callContext) { parse(cc.httpBody.getOrElse("")) } + entries = json match { + case JArray(arr) => arr.collect { case o: JObject => o } + case o: JObject => List(o) + case _ => Nil + } + _ <- Helper.booleanToFuture(s"$InvalidJsonFormat each access entry needs a non-empty user_id", 400, cc = callContext) { + entries.nonEmpty && entries.forall(o => strField(o, "user_id").isDefined) + } + _ = entries.foreach { o => + aclVend.grant(id, strField(o, "user_id").get, + canRead = boolField(o, "can_read", default = false), + canUpdate = boolField(o, "can_update", default = false), + canDelete = boolField(o, "can_delete", default = false), + canGrant = boolField(o, "can_grant", default = true), // §8.1: re-share by default + entityName, bankId, grantedBy = granter.userId) + } + } yield rowAccessListJson(id) + } + + private def revokeRowAccess(req: Request[IO], bankId: Option[String], entityName: String, id: String, grantUserId: String): IO[Response[IO]] = + EndpointHelpers.executeAndRespond(req) { cc => + for { + _ <- rowAccessAuthorise(cc, bankId, entityName, id, DELETE) + removed = aclVend.revoke(id, grantUserId).getOrElse(0) // cascades to downstream grants (§7) + } yield (("revoked_count" -> removed): JObject) + } + // ----- generic endpoint (authenticated, system / bank / personal) ----- private def genericGet(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = EndpointHelpers.executeAndRespond(req) { cc => + if (isRowLevel(bankId, entityName)) rowLevelGet(req, cc, bankId, entityName, id) + else _genericGet(req, cc, bankId, entityName, id, isPersonalEntity) + } + + private def _genericGet(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): Future[JValue] = { val isGetAll = StringUtils.isBlank(id) val operation: DynamicEntityOperation = if (isGetAll) GET_ALL else GET_ONE val callContext0 = enrichCallContext(cc, operation, entityName, bankId, if (isPersonalEntity) "my" else "") @@ -313,6 +530,8 @@ object Http4sDynamicEntity extends MdcLoggable { _ <- failIf(afterIntercept(callContext, operationId), callContext) queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty) decision = if (isGetAll) decideProjection(req, bankId, entityName, queryPlan) else UseInMemory + _ <- if (decision == JoinsNeedProjection) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false } + else Future.successful(true) _ <- if (decision == PendingProjection) Helper.booleanToFuture(DynamicEntityFieldNotYetQueryable, 409, cc = callContext) { false } else Future.successful(true) // Projection path: serve the list from SQL, skipping the fetch-all connector call. @@ -355,11 +574,23 @@ object Http4sDynamicEntity extends MdcLoggable { createJson = stripFields(json.asInstanceOf[JObject], writeRestrictedFieldsOf(bankId, entityName)) (box, _) <- NewStyle.function.invokeDynamicConnector(CREATE, entityName, Some(createJson), None, bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) + // Row-level access: bootstrap the owner ACL row (R/U/D + Grant) so the creator can read, + // edit, and share their own record with no role and no meta-admin hop (§4 / §8.1). + _ = if (isRowLevel(bankId, entityName)) (singleObject \ DynamicEntityHelper.createEntityId(entityName)) match { + case JString(rid) => + aclVend.grant(rid, u.userId, canRead = true, canUpdate = true, canDelete = true, canGrant = true, entityName, bankId, grantedBy = u.userId) + case _ => + } } yield wrapBankId(bankId, (singleName(entityName) -> singleObject)) } private def genericPut(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = EndpointHelpers.executeAndRespond(req) { cc => + if (isRowLevel(bankId, entityName)) rowLevelPut(req, cc, bankId, entityName, id) + else _genericPut(req, cc, bankId, entityName, id, isPersonalEntity) + } + + private def _genericPut(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): Future[JValue] = { val callContext0 = enrichCallContext(cc, UPDATE, entityName, bankId, if (isPersonalEntity) "my" else "") val operationId = callContext0.operationId.orNull for { @@ -382,6 +613,11 @@ object Http4sDynamicEntity extends MdcLoggable { private def genericPatch(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = EndpointHelpers.executeAndRespond(req) { cc => + if (isRowLevel(bankId, entityName)) rowLevelPatch(req, cc, bankId, entityName, id) + else _genericPatch(req, cc, bankId, entityName, id, isPersonalEntity) + } + + private def _genericPatch(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): Future[JValue] = { val callContext0 = enrichCallContext(cc, UPDATE, entityName, bankId, if (isPersonalEntity) "my" else "") val operationId = callContext0.operationId.orNull for { @@ -409,6 +645,11 @@ object Http4sDynamicEntity extends MdcLoggable { private def genericDelete(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = EndpointHelpers.executeAndRespond(req) { cc => + if (isRowLevel(bankId, entityName)) rowLevelDelete(req, cc, bankId, entityName, id) + else _genericDelete(req, cc, bankId, entityName, id, isPersonalEntity) + } + + private def _genericDelete(req: Request[IO], cc: CallContext, bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): Future[JValue] = { val callContext0 = enrichCallContext(cc, DELETE, entityName, bankId, if (isPersonalEntity) "my" else "") val operationId = callContext0.operationId.orNull for { @@ -439,6 +680,9 @@ object Http4sDynamicEntity extends MdcLoggable { (_, callContext) <- anonymousAccess(callContext0) (_, callContext) <- bankCheck(bankId, callContext) queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty) + // Public reads are in-memory only; joins require the projection backend. + _ <- if (queryPlan.joins.nonEmpty) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false } + else Future.successful(true) (box, _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, None, Option(id).filter(StringUtils.isNotBlank), bankId, None, None, false, Some(cc)) _ <- Helper.booleanToFuture(notFoundMsg(entityName, id, bankId), 404, cc = callContext) { box.isDefined } } yield { @@ -469,6 +713,9 @@ object Http4sDynamicEntity extends MdcLoggable { _ <- NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canGetRole(entityName, bankId), callContext) _ <- failIf(afterIntercept(callContext, operationId), callContext) queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty) + // Community reads are in-memory only; joins require the projection backend. + _ <- if (queryPlan.joins.nonEmpty) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false } + else Future.successful(true) } yield { if (isGetAll) { val resultList: List[JObject] = DynamicDataProvider.connectorMethodProvider.vend.getAllDataJsonCommunity(bankId, entityName) @@ -500,6 +747,13 @@ object Http4sDynamicEntity extends MdcLoggable { Some(r => publicGet(r, bankId, entityName, id)) case (Method.GET, CommunityEntityName(bankId, entityName, id)) => Some(r => communityGet(r, bankId, entityName, id)) + // Row-level access management — must precede the generic EntityName match. + case (Method.GET, EntityAccessName(bankId, entityName, id, None)) => + Some(r => listRowAccess(r, bankId, entityName, id)) + case (Method.PUT, EntityAccessName(bankId, entityName, id, None)) => + Some(r => upsertRowAccess(r, bankId, entityName, id)) + case (Method.DELETE, EntityAccessName(bankId, entityName, id, Some(grantUserId))) => + Some(r => revokeRowAccess(r, bankId, entityName, id, grantUserId)) case (method, EntityName(bankId, entityName, id, isPersonalEntity)) => method match { case Method.GET => Some(r => genericGet(r, bankId, entityName, id, isPersonalEntity)) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 2ba96a112e..4b0d6c1f1a 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -118,11 +118,37 @@ object CommunityEntityName { } } +/** + * Row-level access (ACL) management routes: `...///access[/]`. + * Result: (bankId, entityName, dataId, grantedUserIdOpt). Matches whenever the entity exists + * (row-level or not); the handler returns 400 when the entity isn't row-level. See §6. + */ +object EntityAccessName { + private def entityExists(bankId: Option[String], entityName: String): Boolean = + DynamicEntityHelper.definitionsMap.exists { case ((b, n), info) => b == bankId && n == entityName && info.bankId == bankId } + + def unapply(url: List[String]): Option[(Option[String], String, String, Option[String])] = url match { + //eg: /FooBar21/FOO_BAR21_ID/access + case entityName :: id :: "access" :: Nil if entityExists(None, entityName) => + Some((None, entityName, id, None)) + //eg: /FooBar21/FOO_BAR21_ID/access/USER_ID + case entityName :: id :: "access" :: userId :: Nil if entityExists(None, entityName) => + Some((None, entityName, id, Some(userId))) + //eg: /banks/BANK_ID/FooBar21/FOO_BAR21_ID/access + case "banks" :: bankId :: entityName :: id :: "access" :: Nil if entityExists(Some(bankId), entityName) => + Some((Some(bankId), entityName, id, None)) + //eg: /banks/BANK_ID/FooBar21/FOO_BAR21_ID/access/USER_ID + case "banks" :: bankId :: entityName :: id :: "access" :: userId :: Nil if entityExists(Some(bankId), entityName) => + Some((Some(bankId), entityName, id, Some(userId))) + case _ => None + } +} + object DynamicEntityHelper { private val implementedInApiVersion = ApiVersion.v4_0_0 // (Some(BankId), EntityName, DynamicEntityInfo) - def definitionsMap: Map[(Option[String], String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((it.bankId, it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole))).toMap + def definitionsMap: Map[(Option[String], String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((it.bankId, it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess))).toMap def dynamicEntityRoles: List[String] = NewStyle.function.getDynamicEntities(None, true).flatMap { dEntity => val baseRoles = DynamicEntityInfo.roleNames(dEntity.entityName, dEntity.bankId) @@ -237,9 +263,7 @@ object DynamicEntityHelper { | |${userAuthenticationMessage(true)} | - |Can do filter on the fields - |e.g: /${entityName}?name=James%20Brown&number=123.456&number=11.11 - |Will do filter by this rule: name == "James Brown" && (number==123.456 || number=11.11) + |${dynamicEntityInfo.listQueryDoc(joinsSupported = true)} |""".stripMargin, EmptyBody, dynamicEntityInfo.getExampleList, @@ -420,9 +444,7 @@ object DynamicEntityHelper { | |${userAuthenticationMessage(true)} | - |Can do filter on the fields - |e.g: /${entityName}?name=James%20Brown&number=123.456&number=11.11 - |Will do filter by this rule: name == "James Brown" && (number==123.456 || number=11.11) + |${dynamicEntityInfo.listQueryDoc(joinsSupported = true)} |""".stripMargin, EmptyBody, dynamicEntityInfo.getExampleList, @@ -572,9 +594,7 @@ object DynamicEntityHelper { | |Authentication is Optional | - |Can do filter on the fields - |e.g: /${entityName}?name=James%20Brown&number=123.456&number=11.11 - |Will do filter by this rule: name == "James Brown" && (number==123.456 || number=11.11) + |${dynamicEntityInfo.listQueryDoc(joinsSupported = false)} |""".stripMargin, EmptyBody, dynamicEntityInfo.getExampleList, @@ -630,9 +650,7 @@ object DynamicEntityHelper { | |Authentication is Required | - |Can do filter on the fields - |e.g: /${entityName}?name=James%20Brown&number=123.456&number=11.11 - |Will do filter by this rule: name == "James Brown" && (number==123.456 || number=11.11) + |${dynamicEntityInfo.listQueryDoc(joinsSupported = false)} |""".stripMargin, EmptyBody, dynamicEntityInfo.getExampleList, @@ -724,7 +742,7 @@ object DynamicEntityHelper { |""".stripMargin } -case class DynamicEntityInfo(definition: String, entityName: String, bankId: Option[String], hasPersonalEntity: Boolean, hasPublicAccess: Boolean = false, hasCommunityAccess: Boolean = false, personalRequiresRole: Boolean = false) { +case class DynamicEntityInfo(definition: String, entityName: String, bankId: Option[String], hasPersonalEntity: Boolean, hasPublicAccess: Boolean = false, hasCommunityAccess: Boolean = false, personalRequiresRole: Boolean = false, useRowLevelAccess: Boolean = false) { import com.openbankproject.commons.util.json import code.api.dynamic.entity.query.FieldSpec @@ -838,6 +856,7 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt val canUpdateRole: ApiRole = DynamicEntityInfo.canUpdateRole(entityName, bankId) val canGetRole: ApiRole = DynamicEntityInfo.canGetRole(entityName, bankId) val canDeleteRole: ApiRole = DynamicEntityInfo.canDeleteRole(entityName, bankId) + val canGrantRowAccessRole: ApiRole = DynamicEntityInfo.canGrantRowAccessRole(entityName, bankId) // ----- Field-level access control (mirrors DynamicEntityT; here `entity` is already the per-entity object) ----- private def restrictedFields(requiredFlag: String, roleKey: String): List[String] = @@ -865,18 +884,92 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt /** * Fields declared `indexed` (DE_indexing): name -> (declared type, index kind "scalar"|"spatial"). - * Only recognised DynamicEntityFieldType fields are surfaced; this is the queryable allow-list the - * planner validates against. (Reference-typed indexed fields are not yet supported.) + * A `reference:` field surfaces as [[DynamicEntityFieldType.reference]] (its value is an id + * String, so it indexes/queries like a string); the target entity is tracked in [[referenceFields]]. + * Other unrecognised types are dropped — this is the queryable allow-list the planner validates against. */ lazy val indexedFields: Map[String, FieldSpec] = (entity \ "properties") match { case props: JObject => props.obj.collect { case JField(name, propDef: JObject) if (propDef \ "indexed") == JBool(true) => val typeName = (propDef \ "type") match { case JString(s) => s; case _ => "" } val kind = (propDef \ "index") match { case JString(s) => s; case _ => "scalar" } - DynamicEntityFieldType.withNameOption(typeName).map(ft => name -> FieldSpec(ft, kind)) + val fieldTypeOpt = + if (typeName.startsWith("reference:")) Some(DynamicEntityFieldType.reference) + else DynamicEntityFieldType.withNameOption(typeName) + fieldTypeOpt.map(ft => name -> FieldSpec(ft, kind)) }.flatten.toMap case _ => Map.empty } + + /** + * Indexed `reference:` fields: fieldName -> target entity name (the part after "reference:"). + * The join planner uses this to resolve one-hop EXISTS/NOT EXISTS edges between entities; only + * declared reference fields are joinable (a plain string field holding ids is not). See + * ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + */ + lazy val referenceFields: Map[String, String] = (entity \ "properties") match { + case props: JObject => props.obj.collect { + case JField(name, propDef: JObject) + if (propDef \ "indexed") == JBool(true) && + ((propDef \ "type") match { case JString(s) => s.startsWith("reference:"); case _ => false }) => + val target = ((propDef \ "type"): @unchecked) match { case JString(s) => s.stripPrefix("reference:") } + name -> target + }.toMap + case _ => Map.empty + } + + /** + * Human-facing documentation of the list-endpoint query grammar (filter / sort / paginate / one-hop + * joins), appended to the GET-all ResourceDoc descriptions. `joinsSupported` is true on the + * authenticated + /my/ endpoints (which can use the SQL projection backend) and false on the public / + * community endpoints (in-memory only — a join there returns 400). See + * ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + */ + def listQueryDoc(joinsSupported: Boolean): String = { + val indexedNames = indexedFields.keys.toList.sorted + val indexedHint = + if (indexedNames.isEmpty) "_(no fields on this entity are declared `indexed`, so only the legacy bare-parameter filter below is available.)_" + else "Queryable (declared `indexed`) fields: " + indexedNames.mkString("`", "`, `", "`") + "." + val refHint = + if (referenceFields.isEmpty) "" + else "\n\nThis entity's reference fields (each usable as a one-hop join edge to its target entity): " + + referenceFields.toList.sortBy(_._1).map { case (f, t) => s"`$f` → `$t`" }.mkString(", ") + "." + + val base = + s"""**Filtering, sorting and pagination** on the list endpoint: + | + |$indexedHint + | + |* **Filter**: `?obp_filter[FIELD]=OP:VALUE`. Operators: `eq`, `ne`, `in`, `lt`, `gt`, `le`, `ge`, `between`, `like`, `is_null`, `not_set`. `in` / `between` take comma-separated values; `is_null` / `not_set` take no value. Repeat the same key to AND several constraints on one field. + | e.g. `?obp_filter[status]=eq:active&obp_filter[amount]=between:10,100` or `?obp_filter[closed_date]=not_set` + |* **Legacy filter** (still supported): bare field parameters, e.g. `?name=James%20Brown&number=123.456&number=11.11` filters by `name == "James Brown" && (number == 123.456 || number == 11.11)`. + |* **Sort**: `?obp_sort_by=FIELD[,FIELD2]&obp_sort_direction=ASC|DESC`. + |* **Paginate**: `?obp_limit=20&obp_offset=40`. + | + |Filtering, sorting, pagination and `is_null` / `not_set` work on every deployment (no special backend required).""".stripMargin + + val joins = + if (!joinsSupported) + "\n\nOne-hop join queries (`obp_exists` / `obp_not_exists`) are not available on this endpoint." + else + s""" + | + |**One-hop joins (EXISTS / NOT EXISTS)** — filter this entity by a condition on a *related* entity linked through a declared `reference:` field: + | + |* `?obp_exists[RelatedEntity]` — keep rows that have at least one related RelatedEntity. + |* `?obp_exists[RelatedEntity]=filter[FIELD]=eq:VALUE` — ...that have at least one matching the predicate. + |* `?obp_not_exists[RelatedEntity]` — keep rows with NO related RelatedEntity at all. + |* `?obp_not_exists[RelatedEntity]=filter[FIELD]=eq:VALUE` — ...with no related RelatedEntity matching the predicate (this **includes** rows that have no related RelatedEntity). + |* If two entities are linked by more than one reference, disambiguate the edge with `via:`, e.g. `?obp_exists[RelatedEntity]=via:buyer_ref;filter[status]=eq:signed`. + | + |The nested `filter[...]` reuses the operator grammar above. Mind the distinction: `obp_not_exists[X]=filter[flag]=eq:true` (no related X with flag=true — includes rows with no X) is **not** the same as `obp_exists[X]=filter[flag]=ne:true` (has a related X with flag≠true — excludes rows with no X). + | + |Only a field typed `reference:` **and** declared `indexed: true` forms a join edge; a plain string field holding ids is not joinable.$refHint + | + |**Joins require the SQL projection backend** (`dynamic_entity.indexing.backend=auto` on Postgres or SQL Server) with the involved fields indexed and ready. On an in-memory deployment a join query returns `400 (OBP-09022)`; while an index is still building it returns `409`. The plain filter / sort / paginate / `is_null` / `not_set` features above do **not** need the projection backend.""".stripMargin + + base + joins + } } object DynamicEntityInfo { @@ -903,11 +996,21 @@ object DynamicEntityInfo { else getOrCreateDynamicApiRole("CanDeleteDynamicEntity_System" + entityName, false) + // Admin override for row-level access (§3): a holder may grant/list/revoke per-row ACL + // on any row of the entity, even rows they cannot read. Ordinary owner-driven sharing + // does not need this role — it goes through the row's own ACL CanGrant (§8.1). + def canGrantRowAccessRole(entityName: String, bankId:Option[String]): ApiRole = + if(bankId.isDefined) + getOrCreateDynamicApiRole("CanGrantDynamicEntityRowAccess_" + entityName, true) + else + getOrCreateDynamicApiRole("CanGrantDynamicEntityRowAccess_System" + entityName, false) + def roleNames(entityName: String, bankId:Option[String]): List[String] = List( canCreateRole(entityName, bankId), canUpdateRole(entityName, bankId), canGetRole(entityName, bankId), - canDeleteRole(entityName, bankId) + canDeleteRole(entityName, bankId), + canGrantRowAccessRole(entityName, bankId) ).map(_.toString()) // Field-level roles. If the definition declares an explicit write_role/read_role, use it verbatim diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala index de9356af05..c3978f2a02 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala @@ -2,7 +2,7 @@ package code.api.dynamic.entity.projection import cats.effect.IO import code.api.dynamic.entity.helper.DynamicEntityHelper -import code.api.dynamic.entity.query.{DynamicEntityQueryBackend, QueryPlan} +import code.api.dynamic.entity.query.{DynamicEntityQueryBackend, JoinClause, Page, QueryPlan, Quantifier} import doobie._ import doobie.implicits._ import org.json4s.JsonAST.JObject @@ -32,10 +32,19 @@ object PostgresProjectionBackend extends DynamicEntityQueryBackend { def columnOf(f: String): Option[String] = indexed.get(f).map(_ => s"$P." + ProjectionNaming.columnName(f)) def sqlTypeOf(f: String): Option[String] = indexed.get(f).map(s => ProjectionDDL.sqlColumnType(s.fieldType.toString)) - (ProjectionSql.predicates(plan, columnOf, sqlTypeOf), ProjectionSql.orderBy(plan, columnOf)) match { - case (Some(preds), Some(ords)) => - val scope = ProjectionStore.scope(bankId, entityName, isPersonalEntity, userId, D) - val whereAll = if (preds == Fragment.empty) fr"WHERE" ++ scope else fr"WHERE" ++ scope ++ fr"AND" ++ preds + // Each join compiles to a correlated [NOT] EXISTS subquery; None if any of its fields can't resolve. + // `userId` is the authenticated caller (joins are served only on the authenticated get-all path), used + // for user-scoped ACL evaluation when the child entity is row-level. + val joinFragsRaw = plan.joins.map(j => existsFragment(j, bankId, P, D, userId)) + val joinFragsOpt: Option[List[Fragment]] = if (joinFragsRaw.exists(_.isEmpty)) None else Some(joinFragsRaw.flatten) + + (ProjectionSql.predicates(plan, columnOf, sqlTypeOf), ProjectionSql.orderBy(plan, columnOf), joinFragsOpt) match { + case (Some(preds), Some(ords), Some(joinFrags)) => + val scope = ProjectionStore.scope(bankId, entityName, isPersonalEntity, userId, D) + val condParts = (if (preds == Fragment.empty) Nil else List(preds)) ++ joinFrags + val whereAll = + if (condParts.isEmpty) fr"WHERE" ++ scope + else fr"WHERE" ++ scope ++ fr"AND" ++ ProjectionSql.intercalate(condParts, fr"AND") val q = fr"SELECT" ++ Fragment.const(s"$D.${ProjectionStore.jsonColumn}") ++ fr"FROM" ++ Fragment.const(s"$safeTable $P") ++ @@ -47,4 +56,56 @@ object PostgresProjectionBackend extends DynamicEntityQueryBackend { IO.raiseError(new RuntimeException(s"PostgresProjectionBackend: unresolved field in query plan for $entityName")) } } + + /** + * One-hop correlated `[NOT] EXISTS` subquery for a resolved [[JoinClause]] (see + * ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md). Correlation is on the canonical record id ([[ProjectionStore.idColumn]]), + * since a reference value stores the target record's DynamicDataId. The child projection table `cp` is + * joined only when needed (the link lives on the child, or there is a nested predicate on child columns); + * otherwise only the child blob `cd` (scoped) is touched. When the child entity opts into row-level + * access, an ACL sub-EXISTS restricts the count to rows the caller can read (user-scoped semantics). + * Returns None if any referenced child column can't resolve (caller raises / 409 upstream guards this). + */ + private def existsFragment(join: JoinClause, parentBankId: Option[String], parentProjAlias: String, parentBlobAlias: String, callerUserId: Option[String]): Option[Fragment] = { + val childEntity = join.childEntity + val childIndexed = DynamicEntityHelper.definitionsMap.get((parentBankId, childEntity)).map(_.indexedFields).getOrElse(Map.empty) + val childTable = ProjectionNaming.tableName(parentBankId, childEntity) + val cp = "cp"; val cd = "cd" + def childColumnOf(f: String): Option[String] = childIndexed.get(f).map(_ => s"$cp." + ProjectionNaming.columnName(f)) + def childSqlTypeOf(f: String): Option[String] = childIndexed.get(f).map(s => ProjectionDDL.sqlColumnType(s.fieldType.toString)) + + val childPredsOpt = ProjectionSql.predicates(QueryPlan(join.predicate, Nil, Nil, Page.empty), childColumnOf, childSqlTypeOf) + childPredsOpt.map { childPreds => + val linkCol = ProjectionNaming.columnName(join.linkField) + // onChild: child FK (cp.link) == parent id (D.id). else: parent FK (P.link) == child id (cd.id). + val correlate = + if (join.onChild) Fragment.const(s"$cp.$linkCol = $parentBlobAlias.${ProjectionStore.idColumn}") + else Fragment.const(s"$parentProjAlias.$linkCol = $cd.${ProjectionStore.idColumn}") + val needCp = join.onChild || join.predicate.nonEmpty + val from = + if (needCp) + Fragment.const(s"$childTable $cp") ++ fr"JOIN" ++ Fragment.const(s"${ProjectionStore.blobTable} $cd") ++ + fr"ON" ++ Fragment.const(s"$cd.${ProjectionStore.idColumn} = $cp.data_id") + else + Fragment.const(s"${ProjectionStore.blobTable} $cd") + val childScope = ProjectionStore.scope(parentBankId, childEntity, isPersonalEntity = false, None, cd) + val predFrag = if (childPreds == Fragment.empty) Fragment.empty else fr"AND" ++ childPreds + val aclFrag = childAclFragment(childEntity, parentBankId, cd, callerUserId) + val keyword = if (join.quantifier == Quantifier.NotExists) fr"NOT EXISTS" else fr"EXISTS" + keyword ++ fr"(SELECT 1 FROM" ++ from ++ fr"WHERE" ++ correlate ++ fr"AND" ++ childScope ++ predFrag ++ aclFrag ++ fr")" + } + } + + /** ACL restriction for a row-level child: only rows the caller can read count toward EXISTS / NOT EXISTS. */ + private def childAclFragment(childEntity: String, bankId: Option[String], childBlobAlias: String, callerUserId: Option[String]): Fragment = { + val isRowLevel = DynamicEntityHelper.definitionsMap.get((bankId, childEntity)).exists(_.useRowLevelAccess) + (isRowLevel, callerUserId) match { + case (true, Some(uid)) => + fr"AND EXISTS (SELECT 1 FROM" ++ Fragment.const(s"${ProjectionStore.aclTable} acl") ++ + fr"WHERE" ++ Fragment.const(s"acl.${ProjectionStore.aclDataIdColumn} = $childBlobAlias.${ProjectionStore.idColumn}") ++ + fr"AND" ++ Fragment.const(s"acl.${ProjectionStore.aclUserIdColumn}") ++ fr"=" ++ fr0"$uid" ++ + fr"AND" ++ Fragment.const(s"acl.${ProjectionStore.aclCanReadColumn} = true") ++ fr")" + case _ => Fragment.empty + } + } } diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionSql.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionSql.scala index 6e246460c4..b420fe0788 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionSql.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionSql.scala @@ -55,6 +55,9 @@ object ProjectionSql { private def predicate(f: Filter, columnOf: String => Option[String], sqlTypeOf: String => Option[String]): Option[Fragment] = if (FilterOp.spatial.contains(f.op)) None + // Nullary value-absence ops: an absent field projects to a NULL column, so is_null and not_set both + // compile to `IS NULL` (aliases — see ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md OQ1). No cast / operand. + else if (FilterOp.nullary.contains(f.op)) columnOf(f.field).map(col => Fragment.const(col) ++ fr"IS NULL") else for { col <- columnOf(f.field) sqlType <- sqlTypeOf(f.field) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala index af7a89dc30..8f128d2622 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala @@ -24,6 +24,13 @@ object ProjectionStore { val userIdColumn: String = DynamicData.UserId.dbColumnName val personalColumn: String = DynamicData.IsPersonalEntity.dbColumnName + // Row-level access ACL table (Lift-mapped), for user-scoped EXISTS / NOT EXISTS join evaluation: + // a join onto a row-level child counts only child rows the caller can read. + val aclTable: String = code.DynamicData.DynamicDataAccess.dbTableName + val aclDataIdColumn: String = code.DynamicData.DynamicDataAccess.DynamicDataId.dbColumnName + val aclUserIdColumn: String = code.DynamicData.DynamicDataAccess.UserId.dbColumnName + val aclCanReadColumn: String = code.DynamicData.DynamicDataAccess.CanRead.dbColumnName + /** One indexed field's value for a record: safe column, SQL type, coerced text (None => NULL). */ case class ColumnValue(safeColumn: String, sqlType: String, value: Option[String]) @@ -56,11 +63,14 @@ object ProjectionStore { */ def scope(bankId: Option[String], entityName: String, isPersonalEntity: Boolean, userId: Option[String], alias: String = ""): Fragment = { val p = if (alias.isEmpty) "" else alias + "." + // Bind as Option[String]: a system-level entity has bankId=None, which must bind SQL NULL — `orNull` + // as a plain String trips doobie's non-nullable Put[String] ("oops, null"). Put[Option[String]] + // emits NULL for None, and `IS NOT DISTINCT FROM NULL` is the intended null-safe match. val byEntity = Fragment.const(p + entityNameColumn) ++ fr"=" ++ fr0"$entityName" - val byBank = Fragment.const(p + bankIdColumn) ++ fr"IS NOT DISTINCT FROM" ++ fr0"${bankId.orNull: String}" + val byBank = Fragment.const(p + bankIdColumn) ++ fr"IS NOT DISTINCT FROM" ++ fr0"${bankId: Option[String]}" val byPersonal = Fragment.const(p + personalColumn) ++ fr"=" ++ fr0"$isPersonalEntity" val base = byEntity ++ fr"AND" ++ byBank ++ fr"AND" ++ byPersonal - if (isPersonalEntity) base ++ fr"AND" ++ Fragment.const(p + userIdColumn) ++ fr"IS NOT DISTINCT FROM" ++ fr0"${userId.orNull: String}" + if (isPersonalEntity) base ++ fr"AND" ++ Fragment.const(p + userIdColumn) ++ fr"IS NOT DISTINCT FROM" ++ fr0"${userId: Option[String]}" else base } } diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala index 5d0edc15ec..51b2f9d9e2 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala @@ -50,6 +50,9 @@ object InMemoryQueryExecutor { case _ => false } case Like => toStringValue(jv).exists(s => filter.values.exists(v => s.toLowerCase.contains(v.toLowerCase))) + // Value-absence: matches an absent field or an explicit JSON null — mirrors the projection + // backend, where both an absent field and JSON null project to a NULL column (is_null == not_set). + case IsNull | NotSet => jv match { case JNothing | JNull => true; case _ => false } case _ => false // spatial operators are not evaluable in memory } } diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/OperatorMatrix.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/OperatorMatrix.scala index 0d56b8586b..a0d26ab3d5 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/OperatorMatrix.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/OperatorMatrix.scala @@ -17,10 +17,12 @@ object OperatorMatrix { val SCALAR = "scalar" val SPATIAL = "spatial" - private val numericOps: Set[FilterOp] = Set(Eq, Ne, In, Lt, Gt, Le, Ge, Between) - private val dateOps: Set[FilterOp] = Set(Eq, Ne, In, Lt, Gt, Le, Ge, Between) - private val stringOps: Set[FilterOp] = Set(Eq, Ne, In, Like) - private val boolOps: Set[FilterOp] = Set(Eq, Ne) + // Value-absence ops are legal for every non-json type (json is never a plain scalar column). + private val nullOps: Set[FilterOp] = Set(IsNull, NotSet) + private val numericOps: Set[FilterOp] = Set(Eq, Ne, In, Lt, Gt, Le, Ge, Between) ++ nullOps + private val dateOps: Set[FilterOp] = Set(Eq, Ne, In, Lt, Gt, Le, Ge, Between) ++ nullOps + private val stringOps: Set[FilterOp] = Set(Eq, Ne, In, Like) ++ nullOps + private val boolOps: Set[FilterOp] = Set(Eq, Ne) ++ nullOps /** Operators permitted for a field of this type + index kind. Empty = field is not filterable. */ def allowedOps(fieldType: DynamicEntityFieldType, indexKind: String): Set[FilterOp] = diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryModel.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryModel.scala index 84724b8f22..8186b2a419 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryModel.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryModel.scala @@ -24,6 +24,11 @@ object FilterOp { case object Ge extends FilterOp { val name = "ge" } case object Between extends FilterOp { val name = "between" } case object Like extends FilterOp { val name = "like" } + // value-absence (zero operands). is_null and not_set are aliases at the projection layer — an absent + // field and a JSON-null both project to a NULL column, so both compile to SQL `IS NULL` (see + // ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md OQ1). Kept as two names for caller intent / future split. + case object IsNull extends FilterOp { val name = "is_null" } + case object NotSet extends FilterOp { val name = "not_set" } // spatial (served only by a spatial-capable backend; never in-memory) case object Within extends FilterOp { val name = "within" } case object Contains extends FilterOp { val name = "contains" } @@ -31,11 +36,14 @@ object FilterOp { case object DWithin extends FilterOp { val name = "dwithin" } val all: List[FilterOp] = - List(Eq, Ne, In, Lt, Gt, Le, Ge, Between, Like, Within, Contains, Intersects, DWithin) + List(Eq, Ne, In, Lt, Gt, Le, Ge, Between, Like, IsNull, NotSet, Within, Contains, Intersects, DWithin) val byName: Map[String, FilterOp] = all.map(op => op.name -> op).toMap val spatial: Set[FilterOp] = Set(Within, Contains, Intersects, DWithin) + + /** Operators taking no operand. */ + val nullary: Set[FilterOp] = Set(IsNull, NotSet) } sealed trait SortDirection @@ -44,9 +52,39 @@ object SortDirection { case object Desc extends SortDirection } -/** A single filter predicate: `field op values`. `between` carries two values, `in` carries N, others one. */ +/** A single filter predicate: `field op values`. `between` carries two values, `in` carries N, `is_null`/`not_set` zero, others one. */ case class Filter(field: String, op: FilterOp, values: List[String]) +/** EXISTS vs NOT EXISTS for a one-hop join clause. */ +sealed trait Quantifier +object Quantifier { + case object Exists extends Quantifier + case object NotExists extends Quantifier +} + +/** + * A one-hop existence constraint relating the queried (parent) entity to a related (child) entity via a + * declared `reference:` field. Resolved (link field + direction concrete) by the planner. + * childEntity : the related entity's name + * linkField : the reference field carrying the edge + * onChild : true if linkField lives on the child (child references parent); false if on the parent + * predicate : nested scalar filters on the child, AND-composed (may be empty) + * See ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + */ +case class JoinClause( + quantifier: Quantifier, + childEntity: String, + linkField: String, + onChild: Boolean, + predicate: List[Filter] +) + +/** + * A join clause as parsed from query params, before edge resolution: the link field (`via`) and + * direction are not yet known (the planner resolves them from the entity definitions). + */ +case class RawJoin(quantifier: Quantifier, childEntity: String, via: Option[String], predicate: List[Filter]) + /** A single sort key. */ case class SortKey(field: String, direction: SortDirection) @@ -56,8 +94,10 @@ object Page { val empty: Page = Page(None, None) } -/** The fully-parsed, validated query. */ -case class QueryPlan(filters: List[Filter], sort: List[SortKey], page: Page) +/** The fully-parsed, validated query. `joins` are one-hop EXISTS/NOT EXISTS constraints (projection backend only). */ +case class QueryPlan(filters: List[Filter], joins: List[JoinClause], sort: List[SortKey], page: Page) object QueryPlan { - val empty: QueryPlan = QueryPlan(Nil, Nil, Page.empty) + val empty: QueryPlan = QueryPlan(Nil, Nil, Nil, Page.empty) + /** No-join convenience constructor (the common case; in-memory executor never uses joins). */ + def apply(filters: List[Filter], sort: List[SortKey], page: Page): QueryPlan = QueryPlan(filters, Nil, sort, page) } diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala index 3478af33d3..f6489e0207 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala @@ -12,7 +12,12 @@ import scala.util.Try * - filter : `obp_filter[FIELD]=OP:VALUE` (repeat per field; repeat the same key for * multiple constraints on one field). VALUE after the first ':' is opaque; `in` * and `between` split it on commas. OP is required (e.g. `eq:`), which also lets a - * value legitimately contain ':' (only the first ':' is the separator). + * value legitimately contain ':' (only the first ':' is the separator). The nullary + * ops `is_null` / `not_set` take no value (`obp_filter[FIELD]=is_null`). + * - join : `obp_exists[CHILD]` / `obp_not_exists[CHILD]` = `[via:FIELD;] [filter[F]=OP:V][;…]` + * one-hop EXISTS / NOT EXISTS on a related entity. The value is a `;`-separated list + * of an optional `via:` token (link field) plus nested `filter[…]` tokens reusing the + * same operator grammar. See ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. * * This is the one syntax-specific function; everything downstream is syntax-neutral. */ @@ -23,14 +28,19 @@ object QueryParamParser { val SortBy = "obp_sort_by" val SortDirectionKey = "obp_sort_direction" - private val FilterKey = """^obp_filter\[(.+)\]$""".r + private val FilterKey = """^obp_filter\[(.+)\]$""".r + private val ExistsKey = """^obp_exists\[(.+)\]$""".r + private val NotExistsKey = """^obp_not_exists\[(.+)\]$""".r + private val ViaToken = """^via:(.+)$""".r + private val FilterToken = """^filter\[([^\]]+)\]=(.*)$""".r - def parse(params: Map[String, List[String]]): Either[QueryError, (List[Filter], List[SortKey], Page)] = + def parse(params: Map[String, List[String]]): Either[QueryError, (List[Filter], List[RawJoin], List[SortKey], Page)] = for { page <- parsePage(params) sort <- parseSort(params) filters <- parseFilters(params) - } yield (filters, sort, page) + joins <- parseJoins(params) + } yield (filters, joins, sort, page) // ----- pagination ----- @@ -81,13 +91,20 @@ object QueryParamParser { private def parseOneFilter(field: String, raw: String): Either[QueryError, Filter] = { val idx = raw.indexOf(':') if (idx < 0) - Left(QueryError(s"Filter obp_filter[$field] must be of the form :, e.g. obp_filter[$field]=eq:...")) + // No ':' — only a nullary operator (is_null / not_set) is valid in this form. + FilterOp.byName.get(raw) match { + case Some(op) if FilterOp.nullary.contains(op) => Right(Filter(field, op, Nil)) + case _ => + Left(QueryError(s"Filter obp_filter[$field] must be of the form :, e.g. obp_filter[$field]=eq:... (or a nullary op: is_null / not_set).")) + } else { val opToken = raw.substring(0, idx) val operand = raw.substring(idx + 1) FilterOp.byName.get(opToken) match { case None => Left(QueryError(s"Unknown filter operator '$opToken' in obp_filter[$field]. Valid operators: ${FilterOp.byName.keys.toList.sorted.mkString(", ")}.")) + case Some(op) if FilterOp.nullary.contains(op) => + Right(Filter(field, op, Nil)) // nullary: any operand after the ':' is ignored case Some(op) => val vals = if (op == FilterOp.In || op == FilterOp.Between) operand.split(",", -1).toList.map(_.trim) @@ -97,6 +114,34 @@ object QueryParamParser { } } + // ----- join (exists / not_exists) ----- + + private def parseJoins(params: Map[String, List[String]]): Either[QueryError, List[RawJoin]] = { + // NotExistsKey is tried first; `obp_not_exists[...]` never matches `obp_exists[...]` so order is safe either way. + val perKey: List[Either[QueryError, List[RawJoin]]] = params.toList.collect { + case (NotExistsKey(child), values) => traverse(values)(parseOneJoin(Quantifier.NotExists, child, _)) + case (ExistsKey(child), values) => traverse(values)(parseOneJoin(Quantifier.Exists, child, _)) + } + sequence(perKey).map(_.flatten) + } + + private def parseOneJoin(quantifier: Quantifier, child: String, raw: String): Either[QueryError, RawJoin] = { + val keyName = if (quantifier == Quantifier.NotExists) s"obp_not_exists[$child]" else s"obp_exists[$child]" + val tokens = raw.split(";", -1).toList.map(_.trim).filter(_.nonEmpty) + val folded: Either[QueryError, (Option[String], List[Filter])] = + tokens.foldLeft(Right((None, Nil)): Either[QueryError, (Option[String], List[Filter])]) { (acc, tok) => + acc.flatMap { case (via, filters) => + tok match { + case ViaToken(f) => Right((Some(f.trim), filters)) + case FilterToken(f, rest) => parseOneFilter(f.trim, rest).map(flt => (via, filters :+ flt)) + case other => + Left(QueryError(s"Unrecognised token '$other' in $keyName. Expected 'via:' or 'filter[]=:'.")) + } + } + } + folded.map { case (via, filters) => RawJoin(quantifier, child, via, filters) } + } + // ----- helpers ----- private def firstValue(params: Map[String, List[String]], key: String): Option[String] = diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala index 92c9a50696..33a933dafa 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala @@ -8,6 +8,10 @@ import scala.util.Try /** What the planner knows about one declared-`indexed` field. */ case class FieldSpec(fieldType: DynamicEntityFieldType, indexKind: String) +/** What the planner needs to know about a join target (child) entity: its indexed fields (for nested + * predicate validation) and its declared reference fields (for edge inference). */ +case class JoinTargetInfo(indexedFields: Map[String, FieldSpec], referenceFields: Map[String, String]) + /** A contract-layer validation failure (maps to HTTP 400 at the endpoint). */ case class QueryError(message: String) @@ -29,11 +33,76 @@ object QueryPlanner { sort: List[SortKey], page: Page, indexedFields: Map[String, FieldSpec] + ): Either[QueryError, QueryPlan] = + plan(filters, Nil, sort, page, "", indexedFields, Map.empty, _ => None) + + /** + * Full planner including one-hop join clauses. `parentReferenceFields` are the queried entity's + * declared reference fields (for parent→child edges); `childInfoOf` resolves a join-target entity's + * indexed + reference fields (None if no such entity). Joins are resolved to a concrete link field + + * direction here, or rejected with a clear 400. + */ + def plan( + filters: List[Filter], + rawJoins: List[RawJoin], + sort: List[SortKey], + page: Page, + parentEntityName: String, + parentIndexedFields: Map[String, FieldSpec], + parentReferenceFields: Map[String, String], + childInfoOf: String => Option[JoinTargetInfo] ): Either[QueryError, QueryPlan] = for { - _ <- firstError(filters.map(validateFilter(_, indexedFields))) - _ <- firstError(sort.map(validateSort(_, indexedFields))) - } yield QueryPlan(filters, sort, page) + _ <- firstError(filters.map(validateFilter(_, parentIndexedFields))) + _ <- firstError(sort.map(validateSort(_, parentIndexedFields))) + joins <- traverse(rawJoins)(resolveJoin(_, parentEntityName, parentReferenceFields, childInfoOf)) + } yield QueryPlan(filters, joins, sort, page) + + // ----- join resolution ----- + + private def resolveJoin( + raw: RawJoin, + parentEntityName: String, + parentReferenceFields: Map[String, String], + childInfoOf: String => Option[JoinTargetInfo] + ): Either[QueryError, JoinClause] = + childInfoOf(raw.childEntity) match { + case None => Left(QueryError(s"Cannot join '${raw.childEntity}': no such Dynamic Entity.")) + case Some(childInfo) => + // Candidate edges: a child field referencing the parent (onChild=true), or a parent field + // referencing the child (onChild=false). Edge = a declared `reference:` field only. + val childToParent = childInfo.referenceFields.collect { case (f, t) if t == parentEntityName => (f, true) }.toList + val parentToChild = parentReferenceFields.collect { case (f, t) if t == raw.childEntity => (f, false) }.toList + val candidates = childToParent ++ parentToChild + for { + edge <- selectEdge(raw, candidates) + // Nested predicate validates against the CHILD's indexed fields. + _ <- firstError(raw.predicate.map(validateFilter(_, childInfo.indexedFields))) + } yield JoinClause(raw.quantifier, raw.childEntity, edge._1, edge._2, raw.predicate) + } + + private def selectEdge(raw: RawJoin, candidates: List[(String, Boolean)]): Either[QueryError, (String, Boolean)] = + raw.via match { + case Some(field) => + candidates.filter(_._1 == field) match { + case single :: Nil => Right(single) + case Nil => + Left(QueryError(s"No reference field '$field' links '${raw.childEntity}' to the queried entity." + + candidateHint(raw, candidates))) + case _ => Left(QueryError(s"Ambiguous link field '$field' for join with '${raw.childEntity}'.")) + } + case None => + candidates match { + case single :: Nil => Right(single) + case Nil => Left(QueryError(s"Cannot join '${raw.childEntity}': no declared reference links it to the queried entity. " + + "A join edge must be a field typed 'reference:'.")) + case many => Left(QueryError(s"Ambiguous join with '${raw.childEntity}': multiple reference edges " + + s"(${many.map(_._1).mkString(", ")}). Specify via:.")) + } + } + + private def candidateHint(raw: RawJoin, candidates: List[(String, Boolean)]): String = + if (candidates.isEmpty) "" else s" Candidates: ${candidates.map(_._1).mkString(", ")}." // ----- per-term validation ----- @@ -60,6 +129,8 @@ object QueryPlanner { private def arityError(f: Filter): Option[QueryError] = { import FilterOp._ f.op match { + case _ if FilterOp.nullary.contains(f.op) => + if (f.values.nonEmpty) Some(QueryError(s"Operator '${f.op.name}' on '${f.field}' takes no value.")) else None case Between if f.values.size != 2 => Some(QueryError(s"Operator 'between' on '${f.field}' requires exactly two values.")) case In if f.values.isEmpty => Some(QueryError(s"Operator 'in' on '${f.field}' requires at least one value.")) case _ if FilterOp.spatial.contains(f.op) => None // spatial operand shape validated by the spatial backend @@ -68,9 +139,9 @@ object QueryPlanner { } } - /** Scalar values must coerce to the declared type (spatial / like operands are not coerced here). */ + /** Scalar values must coerce to the declared type (spatial / like / nullary operands are not coerced here). */ private def coercionError(f: Filter, spec: FieldSpec): Option[QueryError] = { - if (FilterOp.spatial.contains(f.op) || f.op == FilterOp.Like) None + if (FilterOp.spatial.contains(f.op) || f.op == FilterOp.Like || FilterOp.nullary.contains(f.op)) None else f.values.find(v => !coerces(spec.fieldType, v)) .map(bad => QueryError(s"Value '$bad' is not a valid '${spec.fieldType}' for field '${f.field}'.")) } @@ -87,4 +158,9 @@ object QueryPlanner { private def firstError(results: List[Option[QueryError]]): Either[QueryError, Unit] = results.flatten.headOption.toLeft(()) + + private def traverse[A, B](xs: List[A])(f: A => Either[QueryError, B]): Either[QueryError, List[B]] = + xs.foldRight(Right(Nil): Either[QueryError, List[B]]) { (a, acc) => + for { b <- f(a); rest <- acc } yield b :: rest + } } diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 6d78b396cc..7fcf4e3d3b 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -78,6 +78,9 @@ object ErrorMessages { val DuplicateHeaderKeys = "OBP-09017: Duplicate Header Keys are not allowed." val InvalidDynamicEntityName = "OBP-09018: Invalid entity_name format. Entity names must be lowercase with underscores (snake_case), e.g. 'customer_preferences'. No uppercase letters or spaces allowed." val DynamicEntityFieldNotYetQueryable = "OBP-09019: Requested field(s) are not yet queryable - the index is still being built. Please retry shortly." + val RowLevelAccessRequiresLocalBacking = "OBP-09020: use_row_level_access is only supported for locally-backed dynamic entities. This entity is routed to an external connector (a method routing for dynamicEntityProcess exists for it), where the row-level ACL cannot be enforced. Remove the method routing or disable use_row_level_access." + val RowLevelAccessNotEnabled = "OBP-09021: The row-access endpoints are only available for dynamic entities created with use_row_level_access = true." + val DynamicEntityJoinRequiresProjection = "OBP-09022: obp_exists / obp_not_exists join queries require the SQL projection backend (dynamic_entity.indexing.backend=auto on a supported database). This deployment serves Dynamic Entity reads in-memory, where joins are not supported." // General messages (OBP-10XXX) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index e967a2f790..71423cce2e 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -3318,6 +3318,67 @@ object Glossary extends MdcLoggable { | |For more detailed information about managing Dynamic Entities, see ${getGlossaryItemLink("Dynamic-Entity-Intro")} | +|--- +| +|## Querying the list (GET) endpoint: filter, sort, paginate and one-hop joins +| +|The "Get ... List" endpoint of every Dynamic Entity accepts declarative query parameters: +| +|* **Filter**: `?obp_filter[FIELD]=OP:VALUE`. Operators: `eq`, `ne`, `in`, `lt`, `gt`, `le`, `ge`, `between`, `like`, `is_null`, `not_set`. `in`/`between` take comma-separated values; `is_null`/`not_set` take no value. Repeat the key to AND several constraints on one field. +|* **Sort**: `?obp_sort_by=FIELD[,FIELD2]&obp_sort_direction=ASC` (or DESC). +|* **Paginate**: `?obp_limit=20&obp_offset=40`. +| +|Only fields declared `"indexed": true` are queryable. Filtering, sorting, pagination and `is_null`/`not_set` work on any deployment. +| +|### One-hop joins (EXISTS / NOT EXISTS) +| +|You can filter one entity by a condition on a *related* entity that links to it through a declared `reference:` field — the relational "does a matching related record exist?" question: +| +|* `?obp_exists[child]` — keep parents that have at least one related child. +|* `?obp_exists[child]=filter[status]=eq:active` — parents that have at least one child matching the predicate. +|* `?obp_not_exists[child]` — parents with no related child at all. +|* `?obp_not_exists[child]=filter[status]=eq:active` — parents with no matching child (this **includes** parents that have no child at all). +|* If two entities are linked by more than one reference, disambiguate the edge with `via:`, e.g. `?obp_exists[child]=via:parent_ref;filter[status]=eq:active`. +| +|Mind the distinction: `obp_not_exists[child]=filter[x]=eq:true` (no child with x=true — includes childless parents) is NOT the same as `obp_exists[child]=filter[x]=ne:true` (has a child with x not equal to true — excludes childless parents). +| +|**Requirements for joins:** +| +|* The SQL projection backend must be enabled (`dynamic_entity.indexing.backend=auto` on Postgres or SQL Server). On an in-memory deployment a join query returns `400 (OBP-09022)`; while an index is still building it returns `409`. +|* The link must be a field typed `reference:` AND declared `"indexed": true`. A plain string field holding ids is not joinable. +|* The queried (parent) entity must itself have at least one indexed field. +|* Joins are available on the authenticated and `/my/` list endpoints, not on the public/community ones. +| +|### Worked example: entity `parent` and entity `child` whose field references `parent` +| +|1. Create `parent` with an indexed field (create it first, so `reference:parent` becomes a valid type): +|```json +|POST /obp/v6.0.0/management/system-dynamic-entities +|{ "entity_name": "parent", "has_personal_entity": false, +| "schema": { "properties": { "name": {"type":"string","example":"Acme","indexed":true} } } } +|``` +|2. Create `child` with a reference to `parent`: +|```json +|POST /obp/v6.0.0/management/system-dynamic-entities +|{ "entity_name": "child", "has_personal_entity": false, +| "schema": { "properties": { +| "parent_ref": {"type":"reference:parent","example":"00000000-0000-0000-0000-000000000000","indexed":true}, +| "status": {"type":"string","example":"active","indexed":true} } } } +|``` +|3. Create records. A `POST /obp/v6.0.0/parent` response contains a `parent_id`; put that value into the child's `parent_ref`: +|``` +|POST /obp/v6.0.0/parent {"name":"P1"} -> returns parent_id (call it P1) +|POST /obp/v6.0.0/child {"parent_ref":"","status":"active"} +|POST /obp/v6.0.0/child {"parent_ref":"","status":"closed"} +|``` +|4. Query parents by a condition on their children: +|``` +|GET /obp/v6.0.0/parent?obp_exists[child]=filter[status]=eq:active -> parents that have an active child +|GET /obp/v6.0.0/parent?obp_not_exists[child]=filter[status]=eq:active -> parents with no active child +|GET /obp/v6.0.0/parent?obp_exists[child] -> parents that have any child +|GET /obp/v6.0.0/parent?obp_not_exists[child] -> parents with no child at all +|``` +| """.stripMargin) glossaryItems += GlossaryItem( diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 20fc3ac49c..f05ac16782 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3456,6 +3456,9 @@ object NewStyle extends MdcLoggable{ } yield { if (deleteEntitleMentResult) { DynamicEntityInfo.roleNames(entity.entityName, entity.bankId).foreach(ApiRole.removeDynamicApiRole(_)) + // Cascade row-level ACL rows for this entity (§7) — safety net for any rows not already + // cleaned up by per-row delete; no-op for non-row-level entities. + code.DynamicData.DynamicDataAccessProvider.provider.vend.deleteAllForEntity(entity.entityName, entity.bankId) } deleteEntitleMentResult } diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index fb39bdedd2..09d231f642 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -18,6 +18,31 @@ import java.util.Date import scala.collection.immutable import scala.collection.mutable.HashMap +/** + * ==Schema drift & SQL views — the rule when altering a viewed column== + * + * Postgres refuses `ALTER COLUMN ... TYPE` on a column a view references: + * `ERROR: cannot alter type of a column used by a view or rule`. This is the recurring "schema drift" + * that aborts boot. It bites whenever a view (e.g. `v_account_access_with_views`, `v_consent`, + * `v_metric`) pins a column that a migration — or Lift Schemifier matching a changed model field width — + * wants to alter. + * + * DO NOT fix this by dropping all views on boot/migrate: in a multi-node deployment another node may be + * live and querying those views, so a global drop (even transient) errors the serving node. + * `v_account_access_with_views` and `v_consent` are read by live code. + * + * DO: when you add a migration that alters a column a view references, make that single `runOnce` + * migration do, as one unit (Postgres DDL is transactional, so other nodes never see the view missing): + * DROP VIEW -> ALTER COLUMN ... -> CREATE [OR REPLACE] VIEW ... + * Only drop a view when you must (to alter a column it pins). For a Schemifier-driven width change on a + * viewed column, drop the view in the pre-Schemifier pass (`startedBeforeSchemifier == true`) and + * recreate it in the post-Schemifier pass. + * + * To REPAIR an already-drifted DB forward-only, do NOT edit/duplicate the original migration — `runOnce` + * skips it once logged (`isExecuted`). Add a NEW migration (new name) doing DROP+ALTER+CREATE; it runs + * once to fix existing DBs and is a no-op (CREATE OR REPLACE / IF EXISTS) on fresh ones. The full + * drop-everything reset in running_tests_on_postgres.md is the local recovery path. + */ object Migration extends MdcLoggable { private val migrationScriptsEnabled = ApiPropsWithAlias.migrationScriptsEnabled private val executeAll = getPropsAsBoolValue("migration_scripts.execute_all", false) @@ -122,6 +147,7 @@ object Migration extends MdcLoggable { migrateChatRoomCreatedByAndLastMessageSender() migrateConsentReferenceIdToUuid(startedBeforeSchemifier) migrateMetricConsentReferenceId(startedBeforeSchemifier) + dropFastFirehoseAccountsViews(startedBeforeSchemifier) } private def dummyScript(): Boolean = { @@ -423,6 +449,21 @@ object Migration extends MdcLoggable { } } } + + // Retire the fast-firehose SQL views (firehose -> account directory + ABAC). Runs after the create + // migrations above, so a fresh DB creates-then-drops them and an existing DB just drops them. See + // MigrationOfDropFastFireHoseViews. + private def dropFastFirehoseAccountsViews(startedBeforeSchemifier: Boolean): Boolean = { + if(startedBeforeSchemifier == true) { + logger.warn(s"Migration.database.dropFastFirehoseAccountsViews(true) cannot be run before Schemifier.") + true + } else { + val name = nameOf(dropFastFirehoseAccountsViews(startedBeforeSchemifier)) + runOnce(name) { + MigrationOfDropFastFireHoseViews.dropFastFireHoseViews(name) + } + } + } private def alterUserAuthContextColumnKeyAndValueLength(startedBeforeSchemifier: Boolean): Boolean = { if(startedBeforeSchemifier == true) { @@ -756,6 +797,30 @@ object Migration extends MdcLoggable { } } } + + /** + * Declared max length of a (var)char column, via JDBC metadata (portable across H2/Postgres/MSSQL). + * `None` if the column is absent or has no size (e.g. not a character type). Used by `alterColumn*` + * migrations to ALTER only when the width actually differs — re-issuing `ALTER ... TYPE` to the same + * width is rejected by Postgres when a view references the column (the recurring schema drift). + */ + def columnMaxLength(tableNameLC: String, columnNameLC: String): Option[Int] = { + DB.use(net.liftweb.util.DefaultConnectionIdentifier) { + conn => + val md = conn.getMetaData + val schema = getDefaultSchemaName(conn) + using(md.getColumns(null, schema, null, null)) { rs => + def find(): Option[Int] = + if (!rs.next) None + else if (rs.getString(3).toLowerCase == tableNameLC.toLowerCase && + rs.getString(4).toLowerCase == columnNameLC.toLowerCase) { + val size = rs.getInt(7) // COLUMN_SIZE + if (rs.wasNull) None else Some(size) + } else find() + find() + } + } + } /** * The purpose is to provide answer does a procedure exist at a database instance. */ diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfDropFastFireHoseViews.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfDropFastFireHoseViews.scala new file mode 100644 index 0000000000..2c334797f7 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfDropFastFireHoseViews.scala @@ -0,0 +1,54 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import net.liftweb.mapper.Schemifier + +/** + * Forward migration that removes the fast-firehose SQL objects created by + * [[MigrationOfFastFireHoseView]] / [[MigrationOfFastFireHoseMaterializedView]]. + * + * The firehose approach is being retired in favour of the account directory + ABAC, and neither + * `v_fast_firehose_accounts` nor `mv_fast_firehose_accounts` is read by any application code. They are + * also a recurring schema-drift hazard (a view pinning a column blocks later ALTERs). + * + * `runOnce` (see [[Migration]]) means this executes exactly once per database. It is registered AFTER + * the create migrations in the run sequence, so on a fresh database the views are created and then + * dropped in the same boot; on an existing database the creates are already logged (skipped) and this + * simply drops them. `DROP ... IF EXISTS` makes it a safe no-op when they were never created. + * + * Postgres-only: the create migrations only ever produced working SQL on Postgres (the view body uses + * `string_agg`, and `MATERIALIZED VIEW` is Postgres syntax), so there is nothing to drop on H2 / MS SQL. + */ +object MigrationOfDropFastFireHoseViews { + + def dropFastFireHoseViews(name: String): Boolean = { + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + var isSuccessful = false + + val executedSql = + DbFunction.maybeWrite(true, Schemifier.infoF _) { + APIUtil.getPropsValue("db.driver") openOr ("org.h2.Driver") match { + case value if value.contains("postgresql") => + () => + """ + |DROP MATERIALIZED VIEW IF EXISTS mv_fast_firehose_accounts CASCADE; + |DROP VIEW IF EXISTS v_fast_firehose_accounts CASCADE; + |""".stripMargin + case _ => + () => "" // firehose views were only ever created on Postgres; nothing to drop elsewhere. + } + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Executed SQL: + |$executedSql + |""".stripMargin + isSuccessful = true + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + +} diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfResourceUser.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfResourceUser.scala index ee4f04cf6a..ac17a364bc 100644 --- a/obp-api/src/main/scala/code/api/util/migration/MigrationOfResourceUser.scala +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfResourceUser.scala @@ -61,20 +61,30 @@ object MigrationOfResourceUser { val startDate = System.currentTimeMillis() val commitId: String = APIUtil.gitCommit var isSuccessful = false + val targetLength = 100 + // Idempotent guard: only ALTER when the width actually differs. Re-issuing `ALTER ... TYPE` to + // the SAME width is rejected by Postgres when a view references the column ("cannot alter type of + // a column used by a view or rule") — which is what caused the recurring schema drift on re-runs + // (e.g. test-isolation resets that wipe the migration log but keep the views). See the doc comment + // at the top of Migration.scala. val executedSql = - DbFunction.maybeWrite(true, Schemifier.infoF _) { - APIUtil.getPropsValue("db.driver") match { - case Full(dbDriver) if dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") => - () => - """ALTER TABLE resourceuser ALTER COLUMN email varchar(100); - |""".stripMargin - case _ => - () => - """ALTER TABLE resourceuser ALTER COLUMN email type varchar(100); - |""".stripMargin - } + if (DbFunction.columnMaxLength(ResourceUser._dbTableNameLC, "email").contains(targetLength)) { + s"-- skipped: resourceuser.email already varchar($targetLength)" + } else { + DbFunction.maybeWrite(true, Schemifier.infoF _) { + APIUtil.getPropsValue("db.driver") match { + case Full(dbDriver) if dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") => + () => + """ALTER TABLE resourceuser ALTER COLUMN email varchar(100); + |""".stripMargin + case _ => + () => + """ALTER TABLE resourceuser ALTER COLUMN email type varchar(100); + |""".stripMargin + } + } } val endDate = System.currentTimeMillis() diff --git a/obp-api/src/main/scala/code/api/util/migration/README.md b/obp-api/src/main/scala/code/api/util/migration/README.md new file mode 100644 index 0000000000..85a607c61d --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/README.md @@ -0,0 +1,84 @@ +# Database migrations — and a warning about SQL views + +This package holds the OBP schema migrations (`MigrationOf*.scala`), run on boot by +`Migration.scala`. Each migration is `runOnce` (tracked by name in `MigrationScriptLog`, so it +executes exactly once per database). + +A handful of these migrations create **SQL views**. Views carry a specific, recurring hazard that +everyone touching migrations needs to know about. + +## The SQL views created here + +| View (SQL object) | Created by | Read by app code? | +|---|---|---| +| `v_account_access_with_views` | `MigrationOfAccountAccessWithViewsView` | **Yes** — `DoobieAccountAccessViewQueries` ← `MapperViews` (account access control) | +| `v_consent` | `MigrationOfConsentView` | **Yes** — `DoobieConsentQueries` (consent lookups, v3.1/v4.0/v5.1) | +| `v_metric` | `MigrationOfMetricView` | No — inspection/reporting only | + +The first two are load-bearing: dropping them breaks live request paths. `v_metric` is a convenience +object for looking at the database and is not referenced by any code. + +### Retired views (dropped) + +| View (SQL object) | Created by | Dropped by | +|---|---|---| +| `v_fast_firehose_accounts` | `MigrationOfFastFireHoseView` | `MigrationOfDropFastFireHoseViews` | +| `mv_fast_firehose_accounts` | `MigrationOfFastFireHoseMaterializedView` | `MigrationOfDropFastFireHoseViews` | + +The fast-firehose views have been retired (firehose → account directory + ABAC). The create +migrations still run for historical/`runOnce`-ordering reasons, but `MigrationOfDropFastFireHoseViews` +drops both objects afterwards (`DROP ... IF EXISTS ... CASCADE`, Postgres-only) — so they are not +present on a migrated database. Neither was ever read by application code. + +## ⚠️ The hazard: a view pins the columns it references + +Postgres refuses to change the type of a column a view depends on: + +``` +ERROR: cannot alter type of a column used by a view or rule +DETAIL: rule _RETURN on view depends on column "" +``` + +This fires whenever an `ALTER COLUMN ... TYPE` — from a migration here, or from Lift **Schemifier** +auto-matching a changed model field width on boot — targets a column that a view selects. It is the +recurring **"schema drift"** that aborts boot on long-lived databases. Note it triggers **even when +the type is unchanged** (e.g. re-running `ALTER ... TYPE varchar(100)` on a column already +`varchar(100)`), which is how it surfaces on log-wiping test resets. + +## The rules (authoritative copy lives in `Migration.scala`) + +The full rule is the scaladoc at the top of `object Migration` in `Migration.scala` — read it there. +In short: + +1. **Never drop all views on boot/migrate.** In a multi-node deployment another node is live and + querying these views; a global drop (even transient) errors the serving node. +2. **Prefer an idempotent `ALTER`.** If the column is already the target width, *skip* the alter — it + then never conflicts with a view and no view needs touching. Guard with + `DbFunction.columnMaxLength(table, column).contains(targetWidth)`. Worked example: + `MigrationOfResourceUser.alterColumnEmail`. +3. **Only when the width/type genuinely changes** on a column a view pins: do `DROP VIEW → ALTER COLUMN → CREATE [OR REPLACE] VIEW` inside **one** `runOnce` migration, as a + single statement/transaction (Postgres DDL is transactional, so other nodes never see the view + missing). Don't edit/duplicate the original migration to repair an existing DB — `runOnce` skips a + name already logged; add a **new-named** migration. + +## When adding a column-altering migration + +Check whether a view references the column first: + +```sql +SELECT DISTINCT cl.relname AS dependent_view +FROM pg_attribute a +JOIN pg_class t ON t.oid = a.attrelid AND t.relname = '' +JOIN pg_depend d ON d.refobjid = a.attrelid AND d.refobjsubid = a.attnum +JOIN pg_rewrite rw ON rw.oid = d.objid +JOIN pg_class cl ON cl.oid = rw.ev_class +WHERE a.attname = ''; +``` + +If it returns a view, follow rule 2 (preferred) or rule 3. + +## Recovering a database that has already drifted + +Drop all objects in the schema and let Schemifier + migrations rebuild from scratch (Schemifier +`CREATE`s fresh, so no `ALTER` conflicts). Full procedure: `running_tests_on_postgres.md`. diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index c741e3b57c..d3cd4deeb9 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -457,7 +457,16 @@ object Http4s600 { if (validEntityNamePattern.matcher(entityName).matches()) Future.successful(()) else Future.failed(new RuntimeException(s"$InvalidDynamicEntityName Current value: '$entityName'")) + // §8.5: row-level access is only enforceable for a locally-backed entity. If the entity + // is routed to an external connector (a dynamicEntityProcess method routing keyed on its + // entityName exists), the ACL cannot police its data — reject the combination. + private def localBackingOkForRowLevel(dynamicEntity: DynamicEntityCommons): Boolean = + !dynamicEntity.useRowLevelAccess || + !NewStyle.function.getMethodRoutings(Some("dynamicEntityProcess")) + .exists(_.parameters.exists(p => p.key == "entityName" && p.value == dynamicEntity.entityName)) + private def createDynamicEntityV600(cc: CallContext, dynamicEntity: DynamicEntityCommons) = for { + _ <- Helper.booleanToFuture(RowLevelAccessRequiresLocalBacking, 400, cc = Some(cc)) { localBackingOkForRowLevel(dynamicEntity) } // Wrap the connector call so a thrown RuntimeException (bad schema, etc.) // becomes a 400 InvalidJsonFormat — matches v6 Lift's dispatch wrapper. Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) @@ -473,6 +482,11 @@ object Http4s600 { DynamicEntityInfo.canUpdateRole(result.entityName, dynamicEntity.bankId), DynamicEntityInfo.canGetRole(result.entityName, dynamicEntity.bankId), DynamicEntityInfo.canDeleteRole(result.entityName, dynamicEntity.bankId) + ) ++ ( + // Row-level entities: grant the definition creator the admin row-access role so they + // can bootstrap/administer per-row ACLs across the entity (§8.1 admin override). + if (dynamicEntity.useRowLevelAccess) List(DynamicEntityInfo.canGrantRowAccessRole(result.entityName, dynamicEntity.bankId)) + else Nil ) } yield { crudRoles.foreach(role => @@ -481,6 +495,7 @@ object Http4s600 { } private def updateDynamicEntityV600(cc: CallContext, dynamicEntity: DynamicEntityCommons) = for { + _ <- Helper.booleanToFuture(RowLevelAccessRequiresLocalBacking, 400, cc = Some(cc)) { localBackingOkForRowLevel(dynamicEntity) } Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) .recoverWith { case e: Throwable if !Option(e.getMessage).exists(_.startsWith("OBP-")) => @@ -6783,7 +6798,10 @@ object Http4s600 { | "properties": { | "theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, | "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, - | "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted", "write_role_required": true} + | "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (auto-generated per-field role)", "write_role_required": true}, + | "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role", "write_role": "CanWriteCustomerPreferencesAudit"}, + | "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (auto-generated per-field role)", "read_role_required": true}, + | "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role", "read_role": "CanReadCustomerPreferencesRisk"} | } | } |} @@ -6805,7 +6823,7 @@ object Http4s600 { has_public_access = Some(false), has_community_access = Some(false), personal_requires_role = Some(false), - schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}}}""").asInstanceOf[org.json4s.JsonAST.JObject] + schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}, "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role (writeRole)", "write_role": "CanWriteCustomerPreferencesAudit"}, "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (readRoleRequired)", "read_role_required": true}, "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role (readRole)", "read_role": "CanReadCustomerPreferencesRisk"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), DynamicEntityDefinitionJsonV600( dynamic_entity_id = "abc-123-def", @@ -6816,7 +6834,7 @@ object Http4s600 { has_public_access = false, has_community_access = false, personal_requires_role = false, - schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}}}""").asInstanceOf[org.json4s.JsonAST.JObject] + schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}, "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role (writeRole)", "write_role": "CanWriteCustomerPreferencesAudit"}, "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (readRoleRequired)", "read_role_required": true}, "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role (readRole)", "read_role": "CanReadCustomerPreferencesRisk"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), apiTagManageDynamicEntity :: apiTagApi :: Nil, @@ -6848,7 +6866,10 @@ object Http4s600 { | "properties": { | "theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, | "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, - | "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted", "write_role_required": true} + | "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (auto-generated per-field role)", "write_role_required": true}, + | "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role", "write_role": "CanWriteCustomerPreferencesAudit"}, + | "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (auto-generated per-field role)", "read_role_required": true}, + | "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role", "read_role": "CanReadCustomerPreferencesRisk"} | } | } |} @@ -6870,7 +6891,7 @@ object Http4s600 { has_public_access = Some(false), has_community_access = Some(false), personal_requires_role = Some(false), - schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}}}""").asInstanceOf[org.json4s.JsonAST.JObject] + schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}, "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role (writeRole)", "write_role": "CanWriteCustomerPreferencesAudit"}, "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (readRoleRequired)", "read_role_required": true}, "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role (readRole)", "read_role": "CanReadCustomerPreferencesRisk"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), DynamicEntityDefinitionJsonV600( dynamic_entity_id = "abc-123-def", @@ -6881,7 +6902,7 @@ object Http4s600 { has_public_access = false, has_community_access = false, personal_requires_role = false, - schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}}}""").asInstanceOf[org.json4s.JsonAST.JObject] + schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference"}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (writeRoleRequired)", "write_role_required": true}, "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role (writeRole)", "write_role": "CanWriteCustomerPreferencesAudit"}, "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (readRoleRequired)", "read_role_required": true}, "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role (readRole)", "read_role": "CanReadCustomerPreferencesRisk"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), List( $BankNotFound, diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 8f23aedf25..c704b4fff6 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -953,6 +953,7 @@ case class DynamicEntityDefinitionJsonV600( has_public_access: Boolean = false, has_community_access: Boolean = false, personal_requires_role: Boolean = false, + use_row_level_access: Boolean = false, schema: org.json4s.JsonAST.JObject, _links: Option[DynamicEntityLinksJsonV600] = None ) @@ -971,6 +972,7 @@ case class DynamicEntityDefinitionWithCountJsonV600( has_public_access: Boolean = false, has_community_access: Boolean = false, personal_requires_role: Boolean = false, + use_row_level_access: Boolean = false, schema: org.json4s.JsonAST.JObject, record_count: Long, _links: Option[DynamicEntityLinksJsonV600] = None @@ -987,6 +989,7 @@ case class CreateDynamicEntityRequestJsonV600( has_public_access: Option[Boolean] = None, // defaults to false if not provided has_community_access: Option[Boolean] = None, // defaults to false if not provided personal_requires_role: Option[Boolean] = None, // defaults to false if not provided + use_row_level_access: Option[Boolean] = None, // defaults to false if not provided schema: org.json4s.JsonAST.JObject ) @@ -997,6 +1000,7 @@ case class UpdateDynamicEntityRequestJsonV600( has_public_access: Option[Boolean] = None, has_community_access: Option[Boolean] = None, personal_requires_role: Option[Boolean] = None, + use_row_level_access: Option[Boolean] = None, schema: org.json4s.JsonAST.JObject ) @@ -2541,7 +2545,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { val schemaOption = fullJson.obj.find(_.name == entity.entityName).map(_.value.asInstanceOf[JObject]) // Validate that the dynamic key matches entity_name - val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole") + val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole", "useRowLevelAccess") val dynamicKeyName = fullJson.obj.find(f => !knownFlagFields.contains(f.name)).map(_.name) if (dynamicKeyName.exists(_ != entity.entityName)) { throw new IllegalStateException( @@ -2564,6 +2568,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { has_public_access = entity.hasPublicAccess, has_community_access = entity.hasCommunityAccess, personal_requires_role = entity.personalRequiresRole, + use_row_level_access = entity.useRowLevelAccess, schema = schemaObj, _links = Some(links) ) @@ -2587,7 +2592,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { val schemaOption = fullJson.obj.find(_.name == entity.entityName).map(_.value.asInstanceOf[JObject]) // Validate that the dynamic key matches entity_name - val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole") + val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole", "useRowLevelAccess") val dynamicKeyName = fullJson.obj.find(f => !knownFlagFields.contains(f.name)).map(_.name) if (dynamicKeyName.exists(_ != entity.entityName)) { throw new IllegalStateException( @@ -2610,6 +2615,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { has_public_access = entity.hasPublicAccess, has_community_access = entity.hasCommunityAccess, personal_requires_role = entity.personalRequiresRole, + use_row_level_access = entity.useRowLevelAccess, schema = schema, record_count = recordCount, _links = Some(links) @@ -2641,6 +2647,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { val hasPublicAccess = request.has_public_access.getOrElse(false) val hasCommunityAccess = request.has_community_access.getOrElse(false) val personalRequiresRole = request.personal_requires_role.getOrElse(false) + val useRowLevelAccess = request.use_row_level_access.getOrElse(false) // Build the internal format: entity name as dynamic key + flags JObject( @@ -2649,6 +2656,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { JField("hasPublicAccess", JBool(hasPublicAccess)) :: JField("hasCommunityAccess", JBool(hasCommunityAccess)) :: JField("personalRequiresRole", JBool(personalRequiresRole)) :: + JField("useRowLevelAccess", JBool(useRowLevelAccess)) :: Nil ) } @@ -2660,6 +2668,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { val hasPublicAccess = request.has_public_access.getOrElse(false) val hasCommunityAccess = request.has_community_access.getOrElse(false) val personalRequiresRole = request.personal_requires_role.getOrElse(false) + val useRowLevelAccess = request.use_row_level_access.getOrElse(false) // Build the internal format: entity name as dynamic key + flags JObject( @@ -2668,6 +2677,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { JField("hasPublicAccess", JBool(hasPublicAccess)) :: JField("hasCommunityAccess", JBool(hasCommunityAccess)) :: JField("personalRequiresRole", JBool(personalRequiresRole)) :: + JField("useRowLevelAccess", JBool(useRowLevelAccess)) :: Nil ) } diff --git a/obp-api/src/main/scala/code/dynamicEntity/DynamicDataAccessProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/DynamicDataAccessProvider.scala new file mode 100644 index 0000000000..8199075007 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicEntity/DynamicDataAccessProvider.scala @@ -0,0 +1,70 @@ +package code.DynamicData + +import net.liftweb.common.Box +import net.liftweb.util.SimpleInjector + +/** + * Row-level access control for Dynamic Entities. + * + * When a dynamic entity is defined with `useRowLevelAccess = true`, access to each + * individual data row is decided by an ACL row in `DynamicDataAccess` rather than by the + * per-entity owner/community scope. See ideas/DYNAMIC_ENTITY_ROW_LEVEL_ACCESS.md. + */ +object DynamicDataAccessProvider extends SimpleInjector { + val provider = new Inject(buildOne _) {} + def buildOne: MappedDynamicDataAccessProvider.type = MappedDynamicDataAccessProvider +} + +/** The four per-row permissions an ACL row can confer. */ +sealed trait DynamicDataAccessPermission +object DynamicDataAccessPermission { + case object Read extends DynamicDataAccessPermission + case object Update extends DynamicDataAccessPermission + case object Delete extends DynamicDataAccessPermission + case object Grant extends DynamicDataAccessPermission +} + +trait DynamicDataAccessT { + def dynamicDataId: String + def userId: String + def canRead: Boolean + def canUpdate: Boolean + def canDelete: Boolean + def canGrant: Boolean + def grantedBy: String + def entityName: String + def bankId: Option[String] +} + +trait DynamicDataAccessProvider { + + /** + * Upsert one ACL row: grant (or update) `userId`'s permissions on `dynamicDataId`. + * `grantedBy` records the userId who created the grant, for the revoke cascade. + */ + def grant(dynamicDataId: String, userId: String, + canRead: Boolean, canUpdate: Boolean, canDelete: Boolean, canGrant: Boolean, + entityName: String, bankId: Option[String], grantedBy: String): Box[DynamicDataAccessT] + + /** + * Revoke `userId`'s access to `dynamicDataId` AND cascade: every grant transitively + * made by `userId` on the same row is removed too (walk `grantedBy` with a visited-set + * so re-share cycles terminate). Returns the number of ACL rows removed. + */ + def revoke(dynamicDataId: String, userId: String): Box[Int] + + /** All ACL rows for a single data row (for the GET .../access listing). */ + def getAccessForRow(dynamicDataId: String): List[DynamicDataAccessT] + + /** Ids of the rows of `entityName`/`bankId` that `userId` may read — the get-all filter. */ + def getReadableRowIds(userId: String, entityName: String, bankId: Option[String]): List[String] + + /** Does `userId` hold `permission` on `dynamicDataId`? */ + def allows(dynamicDataId: String, userId: String, permission: DynamicDataAccessPermission): Boolean + + /** Cascade on row delete: remove every ACL row for the data row. */ + def deleteAllForRow(dynamicDataId: String): Box[Boolean] + + /** Cascade on entity delete: remove every ACL row for the entity/bank. */ + def deleteAllForEntity(entityName: String, bankId: Option[String]): Box[Boolean] +} diff --git a/obp-api/src/main/scala/code/dynamicEntity/DynamicDataProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/DynamicDataProvider.scala index 00c02eb6da..9143095d3d 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/DynamicDataProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/DynamicDataProvider.scala @@ -46,6 +46,12 @@ trait DynamicDataProvider { def getAllCommunity(bankId: Option[String], entityName: String): List[DynamicDataT] def getAllDataJsonCommunity(bankId: Option[String], entityName: String): List[JObject] def getCommunity(bankId: Option[String], entityName: String, id: String): Box[DynamicDataT] + + // Community mutation methods - operate on a row regardless of owner (used by row-level access, + // where the ACL, not ownership, decides who may update/delete). Preserve the row's existing + // userId / isPersonalEntity so its provenance is unchanged. + def updateCommunity(bankId: Option[String], entityName: String, requestBody: JObject, id: String): Box[DynamicDataT] + def deleteCommunity(bankId: Option[String], entityName: String, id: String): Box[Boolean] } diff --git a/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala index 8566b6d1a2..3e409928c7 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala @@ -41,6 +41,7 @@ trait DynamicEntityT { def hasPublicAccess: Boolean def hasCommunityAccess: Boolean def personalRequiresRole: Boolean + def useRowLevelAccess: Boolean /** * Add Option(bank_id) to Dynamic Entity. @@ -422,7 +423,8 @@ case class DynamicEntityCommons(entityName: String, hasPersonalEntity: Boolean, hasPublicAccess: Boolean = false, hasCommunityAccess: Boolean = false, - personalRequiresRole: Boolean = false + personalRequiresRole: Boolean = false, + useRowLevelAccess: Boolean = false ) extends DynamicEntityT with JsonFieldReName object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommons] { @@ -467,7 +469,7 @@ object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommo val fields = jsonObject.obj // Known flag field names at the root level (not the entity definition itself) - val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole") + val knownFlagFields = Set("hasPersonalEntity", "hasPublicAccess", "hasCommunityAccess", "personalRequiresRole", "useRowLevelAccess") // validate root object fields val fieldsSize = fields.size @@ -482,6 +484,8 @@ object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommo val hasCommunityAccessValue: Boolean = fields.filter(_.name == "hasCommunityAccess").map(_.value.asInstanceOf[JBool].values).headOption.getOrElse(false) // Determine the value of personalRequiresRole; use the field's boolean value if provided, otherwise default to false val personalRequiresRoleValue: Boolean = fields.filter(_.name == "personalRequiresRole").map(_.value.asInstanceOf[JBool].values).headOption.getOrElse(false) + // Determine the value of useRowLevelAccess; use the field's boolean value if provided, otherwise default to false + val useRowLevelAccessValue: Boolean = fields.filter(_.name == "useRowLevelAccess").map(_.value.asInstanceOf[JBool].values).headOption.getOrElse(false) checkFormat(fields.nonEmpty, s"$DynamicEntityInstanceValidateFail The Json root object should have a single entity, but current have none.") checkFormat(entityFields.size == 1, s"$DynamicEntityInstanceValidateFail The Json root object should have exactly one entity field (plus optional flags: ${knownFlagFields.mkString(", ")}), but current root objects: ${fields.map(_.name).mkString(", ")}") @@ -489,6 +493,11 @@ object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommo flagFields.forall(f => knownFlagFields.contains(f.name)), s"$DynamicEntityInstanceValidateFail Unknown flag fields. Allowed flags: ${knownFlagFields.mkString(", ")}. Current root objects: ${fields.map(_.name).mkString(", ")}" ) + // §8.3: useRowLevelAccess is a master switch — mutually exclusive with public/community access. + checkFormat( + !(useRowLevelAccessValue && (hasPublicAccessValue || hasCommunityAccessValue)), + s"$DynamicEntityInstanceValidateFail useRowLevelAccess cannot be combined with hasPublicAccess or hasCommunityAccess." + ) val JField(entityName, metadataJson) = entityFields.head @@ -640,10 +649,13 @@ object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommo } }) - DynamicEntityCommons(entityName, compactRender(jsonObject), dynamicEntityId, userId, bankId, hasPersonalEntityValue, hasPublicAccessValue, hasCommunityAccessValue, personalRequiresRoleValue) + DynamicEntityCommons(entityName, compactRender(jsonObject), dynamicEntityId, userId, bankId, hasPersonalEntityValue, hasPublicAccessValue, hasCommunityAccessValue, personalRequiresRoleValue, useRowLevelAccessValue) } - private def allowedFieldType: List[String] = DynamicEntityFieldType.values.map(_.toString) ++: ReferenceType.referenceTypeNames + // `reference` is an internal query-layer type (see DynamicEntityFieldType.reference), never declared + // bare by callers — they declare `reference:`, which ReferenceType.referenceTypeNames supplies. + private def allowedFieldType: List[String] = + DynamicEntityFieldType.values.filterNot(_ == DynamicEntityFieldType.reference).map(_.toString) ++: ReferenceType.referenceTypeNames } /** diff --git a/obp-api/src/main/scala/code/dynamicEntity/MappedDynamicDataAccessProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MappedDynamicDataAccessProvider.scala new file mode 100644 index 0000000000..68a5f2ae38 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicEntity/MappedDynamicDataAccessProvider.scala @@ -0,0 +1,133 @@ +package code.DynamicData + +import net.liftweb.common.Box +import net.liftweb.mapper._ +import net.liftweb.util.Helpers.tryo + +import scala.collection.mutable + +object MappedDynamicDataAccessProvider extends DynamicDataAccessProvider { + + override def grant(dynamicDataId: String, userId: String, + canRead: Boolean, canUpdate: Boolean, canDelete: Boolean, canGrant: Boolean, + entityName: String, bankId: Option[String], grantedBy: String): Box[DynamicDataAccessT] = tryo { + val row = DynamicDataAccess.find( + By(DynamicDataAccess.DynamicDataId, dynamicDataId), + By(DynamicDataAccess.UserId, userId) + ).getOrElse(DynamicDataAccess.create.DynamicDataId(dynamicDataId).UserId(userId)) + row.CanRead(canRead) + .CanUpdate(canUpdate) + .CanDelete(canDelete) + .CanGrant(canGrant) + .EntityName(entityName) + .BankId(bankId.getOrElse(null)) + .GrantedBy(grantedBy) + .saveMe() + } + + override def revoke(dynamicDataId: String, userId: String): Box[Int] = tryo { + // Walk the GrantedBy edges within this single data row: remove the target user and + // everyone they granted downstream. The visited-set makes re-share cycles terminate + // and absorbs the owner row's self-edge (GrantedBy == UserId). + val toRemove = mutable.LinkedHashSet[String](userId) + val visited = mutable.Set[String]() + var frontier = List(userId) + while (frontier.nonEmpty) { + val current = frontier.head + frontier = frontier.tail + if (!visited.contains(current)) { + visited += current + val children = DynamicDataAccess.findAll( + By(DynamicDataAccess.DynamicDataId, dynamicDataId), + By(DynamicDataAccess.GrantedBy, current) + ).map(_.UserId.get).filterNot(visited.contains) + children.foreach { child => + toRemove += child + frontier = child :: frontier + } + } + } + toRemove.toList.flatMap { uid => + DynamicDataAccess.findAll( + By(DynamicDataAccess.DynamicDataId, dynamicDataId), + By(DynamicDataAccess.UserId, uid) + ) + }.map(_.delete_!).count(identity) + } + + override def getAccessForRow(dynamicDataId: String): List[DynamicDataAccessT] = + DynamicDataAccess.findAll(By(DynamicDataAccess.DynamicDataId, dynamicDataId)) + + override def getReadableRowIds(userId: String, entityName: String, bankId: Option[String]): List[String] = { + val base: List[QueryParam[DynamicDataAccess]] = List( + By(DynamicDataAccess.UserId, userId), + By(DynamicDataAccess.EntityName, entityName), + By(DynamicDataAccess.CanRead, true) + ) + val scoped = bankId match { + case Some(b) => By(DynamicDataAccess.BankId, b) :: base + case None => NullRef(DynamicDataAccess.BankId) :: base + } + DynamicDataAccess.findAll(scoped: _*).map(_.DynamicDataId.get) + } + + override def allows(dynamicDataId: String, userId: String, permission: DynamicDataAccessPermission): Boolean = { + import DynamicDataAccessPermission._ + DynamicDataAccess.find( + By(DynamicDataAccess.DynamicDataId, dynamicDataId), + By(DynamicDataAccess.UserId, userId) + ).map { row => + permission match { + case Read => row.CanRead.get + case Update => row.CanUpdate.get + case Delete => row.CanDelete.get + case Grant => row.CanGrant.get + } + }.getOrElse(false) + } + + override def deleteAllForRow(dynamicDataId: String): Box[Boolean] = tryo { + DynamicDataAccess.findAll(By(DynamicDataAccess.DynamicDataId, dynamicDataId)).forall(_.delete_!) + } + + override def deleteAllForEntity(entityName: String, bankId: Option[String]): Box[Boolean] = tryo { + val params: List[QueryParam[DynamicDataAccess]] = bankId match { + case Some(b) => List(By(DynamicDataAccess.EntityName, entityName), By(DynamicDataAccess.BankId, b)) + case None => List(By(DynamicDataAccess.EntityName, entityName), NullRef(DynamicDataAccess.BankId)) + } + DynamicDataAccess.findAll(params: _*).forall(_.delete_!) + } +} + +class DynamicDataAccess extends DynamicDataAccessT with LongKeyedMapper[DynamicDataAccess] with IdPK { + + override def getSingleton = DynamicDataAccess + + object DynamicDataId extends MappedString(this, 255) + object UserId extends MappedString(this, 255) + object CanRead extends MappedBoolean(this) + object CanUpdate extends MappedBoolean(this) + object CanDelete extends MappedBoolean(this) + object CanGrant extends MappedBoolean(this) + object GrantedBy extends MappedString(this, 255) + object EntityName extends MappedString(this, 255) + object BankId extends MappedString(this, 255) + + override def dynamicDataId: String = DynamicDataId.get + override def userId: String = UserId.get + override def canRead: Boolean = CanRead.get + override def canUpdate: Boolean = CanUpdate.get + override def canDelete: Boolean = CanDelete.get + override def canGrant: Boolean = CanGrant.get + override def grantedBy: String = GrantedBy.get + override def entityName: String = EntityName.get + override def bankId: Option[String] = Option(BankId.get) +} + +object DynamicDataAccess extends DynamicDataAccess with LongKeyedMetaMapper[DynamicDataAccess] { + override def dbIndexes = + UniqueIndex(DynamicDataId, UserId) :: + Index(UserId, EntityName, BankId) :: + Index(DynamicDataId, GrantedBy) :: + super.dbIndexes +} diff --git a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala index f1213be0ce..6aacf6298b 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala @@ -177,6 +177,23 @@ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonForm } } + override def updateCommunity(bankId: Option[String], entityName: String, requestBody: JObject, id: String): Box[DynamicDataT] = { + val dynamicData = getCommunity(bankId, entityName, id) + .openOrThrowException(s"$DynamicDataNotFound dynamicEntityName=$entityName, dynamicDataId=$id") + .asInstanceOf[DynamicData] + // Preserve the row's existing owner/personal flag — row-level access changes the data, not provenance. + saveOrUpdate(bankId, entityName, requestBody, Option(dynamicData.UserId.get), dynamicData.IsPersonalEntity.get, dynamicData) + } + + override def deleteCommunity(bankId: Option[String], entityName: String, id: String): Box[Boolean] = { + getCommunity(bankId, entityName, id).map { d => + val result = d.asInstanceOf[DynamicData].delete_! + // DE_indexing: remove the projection row in the same transaction (no-op unless projection enabled+ready). + code.api.dynamic.entity.projection.ProjectionDualWrite.onDelete(bankId, entityName, id) + result + } + } + override def existsData(bankId: Option[String], dynamicEntityName: String, userId: Option[String], isPersonalEntity: Boolean): Boolean = { if(bankId.isEmpty && !isPersonalEntity){//isPersonalEntity == false, get all the data, no need for specific userId. DynamicData.find( diff --git a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala index d9910c18df..ed37d8a6c3 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala @@ -59,6 +59,24 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson case Full(dynamicEntity) => dynamicEntity } + // §8.4: switching useRowLevelAccess on for an entity that already has rows makes those + // rows admin-only (no backfill). Warn so the operator grants access deliberately. + val wasRowLevel = existsDynamicEntity.map(_.useRowLevelAccess).getOrElse(false) + if (!wasRowLevel && dynamicEntity.useRowLevelAccess) { + val existingRowCount = dynamicEntity.bankId match { + case Some(b) => code.DynamicData.DynamicData.count( + By(code.DynamicData.DynamicData.DynamicEntityName, dynamicEntity.entityName), + By(code.DynamicData.DynamicData.BankId, b)) + case None => code.DynamicData.DynamicData.count( + By(code.DynamicData.DynamicData.DynamicEntityName, dynamicEntity.entityName), + NullRef(code.DynamicData.DynamicData.BankId)) + } + if (existingRowCount > 0) + logger.warn(s"createOrUpdate says: useRowLevelAccess switched on for entity '${dynamicEntity.entityName}' " + + s"(bankId=${dynamicEntity.bankId.getOrElse("none")}) which already has $existingRowCount row(s); these are now " + + s"admin-only until access is granted via the ACL (no backfill — see DYNAMIC_ENTITY_ROW_LEVEL_ACCESS.md §8.4).") + } + tryo{ try { val saved = entityToPersist @@ -70,6 +88,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson .HasPublicAccess(dynamicEntity.hasPublicAccess) .HasCommunityAccess(dynamicEntity.hasCommunityAccess) .PersonalRequiresRole(dynamicEntity.personalRequiresRole) + .UseRowLevelAccess(dynamicEntity.useRowLevelAccess) .saveMe() // DE_indexing: provision/refresh the projection for this definition's indexed scalar fields. // Guarded by projectionEnabled (default off); best-effort (a failure leaves the definition saved @@ -79,7 +98,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson try { val info = code.api.dynamic.entity.helper.DynamicEntityInfo( dynamicEntity.metadataJson, dynamicEntity.entityName, dynamicEntity.bankId, - dynamicEntity.hasPersonalEntity, dynamicEntity.hasPublicAccess, dynamicEntity.hasCommunityAccess, dynamicEntity.personalRequiresRole) + dynamicEntity.hasPersonalEntity, dynamicEntity.hasPublicAccess, dynamicEntity.hasCommunityAccess, dynamicEntity.personalRequiresRole, dynamicEntity.useRowLevelAccess) val scalar = code.api.dynamic.entity.projection.ProjectionProvisioner.scalarFieldsOf(info.indexedFields) if (scalar.nonEmpty) code.api.dynamic.entity.projection.ProjectionProvisioner @@ -124,6 +143,7 @@ class DynamicEntity extends DynamicEntityT with LongKeyedMapper[DynamicEntity] w object HasPublicAccess extends MappedBoolean(this) object HasCommunityAccess extends MappedBoolean(this) object PersonalRequiresRole extends MappedBoolean(this) + object UseRowLevelAccess extends MappedBoolean(this) override def dynamicEntityId: Option[String] = Option(DynamicEntityId.get) override def entityName: String = EntityName.get @@ -134,6 +154,7 @@ class DynamicEntity extends DynamicEntityT with LongKeyedMapper[DynamicEntity] w override def hasPublicAccess: Boolean = HasPublicAccess.get override def hasCommunityAccess: Boolean = HasCommunityAccess.get override def personalRequiresRole: Boolean = PersonalRequiresRole.get + override def useRowLevelAccess: Boolean = UseRowLevelAccess.get } object DynamicEntity extends DynamicEntity with LongKeyedMetaMapper[DynamicEntity] { diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala new file mode 100644 index 0000000000..de91eb3ff3 --- /dev/null +++ b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala @@ -0,0 +1,149 @@ +package code.api.dynamic.entity.query + +import com.openbankproject.commons.model.enums.DynamicEntityFieldType +import org.json4s.JsonAST.JObject +import org.scalatest.{FlatSpec, Matchers} + +/** + * Pure unit tests for the one-hop join feature (obp_exists / obp_not_exists) and the value-absence + * operators (is_null / not_set): query-param parsing, definition-driven edge resolution in the planner, + * and in-memory nullary-op evaluation. No server / DB — the EXISTS/NOT EXISTS SQL itself is exercised by + * the Postgres-gated integration suite. See ideas/DYNAMIC_ENTITY_JOIN_QUERIES.md. + * + * Domain: parent `Partner` (tier), child `Contract` (active, status, partner_id : reference:Partner). + */ +class JoinQuerySpec extends FlatSpec with Matchers { + + private def params(kvs: (String, String)*): Map[String, List[String]] = + kvs.groupBy(_._1).map { case (k, vs) => k -> vs.map(_._2).toList } + + private def rec(s: String): JObject = com.openbankproject.commons.util.JsonAliases.parse(s).asInstanceOf[JObject] + + // ----- planner fixtures ----- + + private val partnerIndexed: Map[String, FieldSpec] = Map("tier" -> FieldSpec(DynamicEntityFieldType.string, "scalar")) + + // child Contract with a single edge to Partner (partner_id) + private val contractSingleEdge = JoinTargetInfo( + indexedFields = Map( + "active" -> FieldSpec(DynamicEntityFieldType.boolean, "scalar"), + "status" -> FieldSpec(DynamicEntityFieldType.string, "scalar"), + "partner_id" -> FieldSpec(DynamicEntityFieldType.reference, "scalar")), + referenceFields = Map("partner_id" -> "Partner")) + + // child Contract with two edges to Partner (buyer_id, seller_id) + private val contractTwoEdges = contractSingleEdge.copy( + indexedFields = contractSingleEdge.indexedFields - "partner_id" + + ("buyer_id" -> FieldSpec(DynamicEntityFieldType.reference, "scalar")) + + ("seller_id" -> FieldSpec(DynamicEntityFieldType.reference, "scalar")), + referenceFields = Map("buyer_id" -> "Partner", "seller_id" -> "Partner")) + + private def childInfoOf(map: Map[String, JoinTargetInfo])(name: String): Option[JoinTargetInfo] = map.get(name) + + private def planJoin(raw: RawJoin, child: JoinTargetInfo, + parentRefs: Map[String, String] = Map.empty): Either[QueryError, QueryPlan] = + QueryPlanner.plan(Nil, List(raw), Nil, Page.empty, "Partner", + partnerIndexed, parentRefs, childInfoOf(Map("Contract" -> child))) + + // ----- parser: join clauses ----- + + "QueryParamParser" should "parse obp_exists with no predicate (has-any)" in { + val Right((_, joins, _, _)) = QueryParamParser.parse(params("obp_exists[Contract]" -> "")) + joins shouldBe List(RawJoin(Quantifier.Exists, "Contract", None, Nil)) + } + + it should "parse obp_not_exists distinctly from obp_exists" in { + val Right((_, joins, _, _)) = QueryParamParser.parse(params("obp_not_exists[Contract]" -> "")) + joins shouldBe List(RawJoin(Quantifier.NotExists, "Contract", None, Nil)) + } + + it should "parse a nested predicate reusing the filter grammar" in { + val Right((_, joins, _, _)) = QueryParamParser.parse(params("obp_exists[Contract]" -> "filter[active]=eq:true")) + joins shouldBe List(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("active", FilterOp.Eq, List("true"))))) + } + + it should "parse via: plus a nested predicate (semicolon-separated)" in { + val Right((_, joins, _, _)) = QueryParamParser.parse(params("obp_exists[Contract]" -> "via:buyer_id;filter[status]=eq:signed")) + joins shouldBe List(RawJoin(Quantifier.Exists, "Contract", Some("buyer_id"), List(Filter("status", FilterOp.Eq, List("signed"))))) + } + + it should "reject an unrecognised token inside a join clause" in { + QueryParamParser.parse(params("obp_exists[Contract]" -> "bogus")).isLeft shouldBe true + } + + // ----- parser: nullary value-absence operators ----- + + it should "parse is_null / not_set with no operand" in { + val Right((f1, _, _, _)) = QueryParamParser.parse(params("obp_filter[tier]" -> "is_null")) + f1 shouldBe List(Filter("tier", FilterOp.IsNull, Nil)) + val Right((f2, _, _, _)) = QueryParamParser.parse(params("obp_filter[tier]" -> "not_set")) + f2 shouldBe List(Filter("tier", FilterOp.NotSet, Nil)) + } + + // ----- planner: edge resolution ----- + + "QueryPlanner join resolution" should "infer the only reference edge (child -> parent)" in { + val plan = planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), contractSingleEdge) + plan.isRight shouldBe true + plan.right.get.joins shouldBe List(JoinClause(Quantifier.Exists, "Contract", "partner_id", onChild = true, Nil)) + } + + it should "reject an ambiguous join when two edges exist and no via is given" in { + planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), contractTwoEdges).isLeft shouldBe true + } + + it should "resolve the edge when via picks one of several candidates" in { + val plan = planJoin(RawJoin(Quantifier.Exists, "Contract", Some("seller_id"), Nil), contractTwoEdges) + plan.right.get.joins.head.linkField shouldBe "seller_id" + } + + it should "reject a via that is not a real reference edge" in { + planJoin(RawJoin(Quantifier.Exists, "Contract", Some("not_a_ref"), Nil), contractTwoEdges).isLeft shouldBe true + } + + it should "reject a join target that has no reference link to the queried entity" in { + val unrelated = JoinTargetInfo(Map("active" -> FieldSpec(DynamicEntityFieldType.boolean, "scalar")), Map.empty) + planJoin(RawJoin(Quantifier.Exists, "Contract", None, Nil), unrelated).isLeft shouldBe true + } + + it should "reject a join onto a non-existent entity" in { + QueryPlanner.plan(Nil, List(RawJoin(Quantifier.Exists, "Nope", None, Nil)), Nil, Page.empty, "Partner", + partnerIndexed, Map.empty, _ => None).isLeft shouldBe true + } + + it should "resolve a parent -> child edge (link on the parent)" in { + val plan = QueryPlanner.plan(Nil, List(RawJoin(Quantifier.NotExists, "Contract", None, Nil)), Nil, Page.empty, + "Partner", partnerIndexed, Map("favourite_contract" -> "Contract"), + childInfoOf(Map("Contract" -> JoinTargetInfo(Map.empty, Map.empty)))) + plan.right.get.joins shouldBe List(JoinClause(Quantifier.NotExists, "Contract", "favourite_contract", onChild = false, Nil)) + } + + // ----- planner: nested predicate validation against the CHILD ----- + + it should "validate the nested predicate against the child's indexed fields" in { + planJoin(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("active", FilterOp.Eq, List("true")))), contractSingleEdge).isRight shouldBe true + // 'colour' is not indexed on the child + planJoin(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("colour", FilterOp.Eq, List("x")))), contractSingleEdge).isLeft shouldBe true + // gt is illegal on a boolean child field + planJoin(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("active", FilterOp.Gt, List("true")))), contractSingleEdge).isLeft shouldBe true + } + + it should "accept a nullary op (no operand) in the nested predicate" in { + planJoin(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("status", FilterOp.IsNull, Nil))), contractSingleEdge).isRight shouldBe true + } + + it should "reject a nullary op carrying an operand" in { + planJoin(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("status", FilterOp.IsNull, List("x")))), contractSingleEdge).isLeft shouldBe true + } + + // ----- in-memory: is_null / not_set evaluation (aliases: match absent field or JSON null) ----- + + "InMemoryQueryExecutor nullary ops" should "match a missing field and an explicit null, but not a present value" in { + val data = List(rec("""{"tier":"gold"}"""), rec("""{"tier":null}"""), rec("""{"other":1}""")) + val fieldTypes = Map("tier" -> DynamicEntityFieldType.string) + for (op <- List(FilterOp.IsNull, FilterOp.NotSet)) { + val out = InMemoryQueryExecutor.execute(data, QueryPlan(List(Filter("tier", op, Nil)), Nil, Page.empty), fieldTypes) + out.size shouldBe 2 // the null one and the missing one; not the "gold" one + } + } +} diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/query/QuerySpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/query/QuerySpec.scala index 39cc15589f..ccea41de63 100644 --- a/obp-api/src/test/scala/code/api/dynamic/entity/query/QuerySpec.scala +++ b/obp-api/src/test/scala/code/api/dynamic/entity/query/QuerySpec.scala @@ -27,7 +27,7 @@ class QuerySpec extends FlatSpec with Matchers { // ----- QueryParamParser ----- "QueryParamParser" should "parse a scalar filter, sort and pagination" in { - val Right((filters, sort, page)) = QueryParamParser.parse(params( + val Right((filters, _, sort, page)) = QueryParamParser.parse(params( "obp_filter[price]" -> "lt:10", "obp_sort_by" -> "price", "obp_sort_direction" -> "DESC", "obp_limit" -> "20", "obp_offset" -> "40")) filters shouldBe List(Filter("price", FilterOp.Lt, List("10"))) @@ -36,7 +36,7 @@ class QuerySpec extends FlatSpec with Matchers { } it should "split in/between values on commas but keep other operands opaque" in { - val Right((filters, _, _)) = QueryParamParser.parse(params( + val Right((filters, _, _, _)) = QueryParamParser.parse(params( "obp_filter[status]" -> "in:a,b,c", "obp_filter[price]" -> "between:5,10")) filters.toSet shouldBe Set( Filter("status", FilterOp.In, List("a", "b", "c")), diff --git a/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala new file mode 100644 index 0000000000..ad6d075945 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinQueryIntegrationTest.scala @@ -0,0 +1,122 @@ +package code.api.v6_0_0 + +import code.api.dynamic.entity.projection.{IndexingCapabilities, ProjectionProvisioner} +import code.api.dynamic.entity.projection.PostgresProjectionBackend +import code.api.dynamic.entity.query._ +import code.api.util.APIUtil +import code.DynamicData.{DynamicDataAccessProvider, DynamicDataProvider} +import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider} +import cats.effect.unsafe.implicits.global +import net.liftweb.util.StringHelpers +import org.json4s.JsonAST._ + +/** + * Phase 3 integration proof for one-hop join queries (obp_exists / obp_not_exists): exercises the actual + * EXISTS / NOT EXISTS SQL that [[PostgresProjectionBackend.query]] generates, against the real Postgres + * test DB. Validates the three meanings of (non-)existence, has-any / has-none, NULL-safety (correlated + * NOT EXISTS, never NOT IN), and user-scoped ACL evaluation for a row-level child. + * + * Real definitions are registered (so the planner-resolved [[JoinClause]] resolves child columns via + * `definitionsMap`), projections provisioned, and rows written through the real provider (dual-write + * populates the projection). Entity names carry a per-run suffix so reruns never see stale data. + * + * Gated OFF by default (set `test.projection.postgres=true` with a Postgres `db.url`). The unit-level + * parser / planner / in-memory behaviour is covered separately by + * `code.api.dynamic.entity.query.JoinQuerySpec`. + */ +class DynamicEntityJoinQueryIntegrationTest extends V600ServerSetup { + + private def run[A](io: cats.effect.IO[A]): A = io.unsafeRunSync() + private val owner = "itest-owner" + private val userA = "itest-user-a" + private val userB = "itest-user-b" + private val sfx = java.util.UUID.randomUUID().toString.take(8) + private val Partner = s"Partner$sfx" + private val Contract = s"Contract$sfx" + private val Deal = s"Deal$sfx" + private def idField(entity: String): String = StringHelpers.snakify(entity) + "_id" + + private def createDef(entity: String, propsJson: String, rowLevel: Boolean = false): Unit = { + val metadata = s"""{"$entity":{"properties":$propsJson}}""" + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate( + DynamicEntityCommons(entity, metadata, None, owner, None, hasPersonalEntity = false, + hasCommunityAccess = true, useRowLevelAccess = rowLevel) + ).openOrThrowException(s"failed to create definition for $entity") + } + + /** Save a record (explicit id so references are controllable). Returns the record's id (= DynamicDataId). */ + private def saveRec(entity: String, fields: (String, JValue)*): String = { + val id = java.util.UUID.randomUUID().toString + val body = JObject(JField(idField(entity), JString(id)) :: fields.toList.map { case (k, v) => JField(k, v) }) + DynamicDataProvider.connectorMethodProvider.vend.save(None, entity, body, Some(owner), false) + .openOrThrowException(s"failed to save $entity record") + id + } + + private def queryPartnerIds(plan: QueryPlan, asUser: String): Set[String] = { + val pidField = idField(Partner) + PostgresProjectionBackend.query(Partner, None, Some(asUser), isPersonalEntity = false, plan) + .map(_.flatMap(o => (o \ pidField) match { case JString(s) => Some(s); case _ => None }).toSet) + .unsafeRunSync() + } + + private def joinPlan(quantifier: Quantifier, child: String, predicate: List[Filter]): QueryPlan = + QueryPlan(Nil, List(JoinClause(quantifier, child, "partner_ref", onChild = true, predicate)), Nil, Page.empty) + + private val activeTrue = List(Filter("active", FilterOp.Eq, List("true"))) + private val activeNotTrue = List(Filter("active", FilterOp.Ne, List("true"))) + + feature("DE one-hop EXISTS / NOT EXISTS join queries on Postgres") { + scenario("three meanings of (non-)existence, has-any/none, NULL-safety, and user-scoped ACL") { + if (!APIUtil.getPropsAsBoolValue("test.projection.postgres", false) || IndexingCapabilities.vendor != IndexingCapabilities.Postgres) + cancel("Postgres projection integration tests disabled (set test.projection.postgres=true with a Postgres db.url).") + + // --- definitions (Partner must exist before Contract/Deal reference it) --- + createDef(Partner, s"""{"${idField(Partner)}":{"type":"string"},"tier":{"type":"string","indexed":true}}""") + createDef(Contract, s"""{"${idField(Contract)}":{"type":"string"},"partner_ref":{"type":"reference:$Partner","indexed":true},"active":{"type":"boolean","indexed":true}}""") + createDef(Deal, s"""{"${idField(Deal)}":{"type":"string"},"partner_ref":{"type":"reference:$Partner","indexed":true}}""", rowLevel = true) + + // --- write all rows first --- + val p1 = saveRec(Partner, "tier" -> JString("gold")) + val p2 = saveRec(Partner, "tier" -> JString("silver")) + val p3 = saveRec(Partner, "tier" -> JString("bronze")) // no contract at all + + saveRec(Contract, "partner_ref" -> JString(p1), "active" -> JBool(true)) // P1 has an ACTIVE contract + saveRec(Contract, "partner_ref" -> JString(p1), "active" -> JBool(false)) // P1 also has an inactive one + saveRec(Contract, "partner_ref" -> JString(p2), "active" -> JBool(false)) // P2 only inactive + saveRec(Contract, "active" -> JBool(false)) // orphan: partner_ref absent (NULL) + + val d1 = saveRec(Deal, "partner_ref" -> JString(p1)) + saveRec(Deal, "partner_ref" -> JString(p2)) // d2: granted to nobody + DynamicDataAccessProvider.provider.vend.grant(d1, userA, canRead = true, canUpdate = false, + canDelete = false, canGrant = false, entityName = Deal, bankId = None, grantedBy = owner) + + // --- provision AFTER writing, so the backfill populates projections from the blobs. + // (This makes the test independent of dynamic_entity.indexing.backend; provisioning's backfill + + // PostgresProjectionBackend.query do not consult that prop — only test.projection.postgres gates us.) + List(Partner, Contract, Deal).foreach(e => run(ProjectionProvisioner.ensureProvisioned(None, e))) + + // 1. EXISTS, predicate active=true -> partners WITH an active contract + queryPartnerIds(joinPlan(Quantifier.Exists, Contract, activeTrue), owner) shouldBe Set(p1) + + // 2. NOT EXISTS, predicate active=true -> partners with NO active contract (incl. zero-contract P3) + queryPartnerIds(joinPlan(Quantifier.NotExists, Contract, activeTrue), owner) shouldBe Set(p2, p3) + + // 3. EXISTS, predicate active!=true -> partners that HAVE a non-active contract (excludes P3) + queryPartnerIds(joinPlan(Quantifier.Exists, Contract, activeNotTrue), owner) shouldBe Set(p1, p2) + + // 4. EXISTS, no predicate -> partners with ANY contract + queryPartnerIds(joinPlan(Quantifier.Exists, Contract, Nil), owner) shouldBe Set(p1, p2) + + // 5. NOT EXISTS, no predicate -> partners with NO contract at all. + // The orphan contract (NULL partner_ref) must not corrupt this (correlated NOT EXISTS, not NOT IN). + queryPartnerIds(joinPlan(Quantifier.NotExists, Contract, Nil), owner) shouldBe Set(p3) + + val dealExists = joinPlan(Quantifier.Exists, Deal, Nil) + // userA can read D1 -> P1 counts; D2 invisible -> P2 excluded. + queryPartnerIds(dealExists, userA) shouldBe Set(p1) + // userB has no grants -> no readable deals -> no partner matches. + queryPartnerIds(dealExists, userB) shouldBe Set.empty[String] + } + } +} diff --git a/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityRowLevelAccessTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityRowLevelAccessTest.scala new file mode 100644 index 0000000000..1852aece67 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityRowLevelAccessTest.scala @@ -0,0 +1,242 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2025, TESOBE GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH +Osloerstrasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + */ +package code.api.v6_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole._ +import code.entitlement.Entitlement +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonDSL._ +import org.json4s.native.Serialization.write +import org.json4s._ +import com.openbankproject.commons.util.JsonAliases._ +import org.scalatest.Tag + +/** + * Row-level access (use_row_level_access) — see ideas/DYNAMIC_ENTITY_ROW_LEVEL_ACCESS.md. + * Covers: owner bootstrap, per-row read isolation (404 hides), owner sharing with no role, + * per-row update gate (403), revoke cascade, get-all ACL filter, admin-role override, + * flag-off 400, and the §8.3 public/community mutual-exclusion guard. + */ +class DynamicEntityRowLevelAccessTest extends V600ServerSetup { + + object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) + + // ==================== Helpers ==================== + + def createSystemEntity(entityJson: JValue): (Int, JValue) = { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateSystemLevelDynamicEntity.toString) + val request = (v6_0_0_Request / "management" / "system-dynamic-entities").POST <@ (user1) + val response = makePostRequest(request, write(entityJson)) + (response.code, response.body) + } + + def deleteSystemEntity(dynamicEntityId: String): Unit = { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanDeleteSystemLevelDynamicEntity.toString) + makeDeleteRequest((v4_0_0_Request / "management" / "system-dynamic-entities" / dynamicEntityId).DELETE <@ (user1)) + } + + def simpleSchema: JValue = parse( + """ + |{ + | "description": "Row-level access test entity.", + | "required": ["name"], + | "properties": { + | "name": { "type": "string", "maxLength": 40, "minLength": 1, "example": "Test" } + | } + |} + """.stripMargin) + + val entityRowLevel: JValue = + ("entity_name" -> "test_rl") ~ + ("use_row_level_access" -> true) ~ + ("schema" -> simpleSchema) + + // user1 owns the entity definition (and is auto-granted the entity roles incl. the admin grant role). + // user1 creates records (it holds CanCreateDynamicEntity_Systemtest_rl from the definition create). + def createRecord(name: String): String = { + val resp = makePostRequest((dynamicEntity_Request / "test_rl").POST <@ (user1), write(("name" -> name): JValue)) + resp.code should equal(201) + (resp.body \ "test_rl" \ "test_rl_id").extract[String] + } + + // ==================== Feature 1: owner bootstrap & read isolation ==================== + + feature("Feature 1: owner bootstrap & per-row read isolation") { + scenario("1.1: creator reads own record; another user gets 404; list is ACL-filtered", VersionOfApi) { + val (code, body) = createSystemEntity(entityRowLevel) + code should equal(201) + val dynamicEntityId = (body \ "dynamic_entity_id").extract[String] + try { + val id = createRecord("owned by user1") + + When("user1 (owner) GETs the record") + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user1)).code should equal(200) + + When("user2 (no ACL) GETs the record") + Then("it is 404 — an unreadable row is indistinguishable from a missing one") + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user2)).code should equal(404) + + When("user1 GETs the list") + val ownerList = makeGetRequest((dynamicEntity_Request / "test_rl").GET <@ (user1)) + ownerList.code should equal(200) + (ownerList.body \ "test_rl_list").asInstanceOf[JArray].arr.size should equal(1) + + When("user2 GETs the list") + Then("it is empty — user2 can read none of the rows") + val otherList = makeGetRequest((dynamicEntity_Request / "test_rl").GET <@ (user2)) + otherList.code should equal(200) + (otherList.body \ "test_rl_list").asInstanceOf[JArray].arr.size should equal(0) + } finally deleteSystemEntity(dynamicEntityId) + } + } + + // ==================== Feature 2: owner shares with no role; per-row update gate ==================== + + feature("Feature 2: owner shares (no role) and the per-row update gate") { + scenario("2.1: owner grants read → grantee reads; update needs a separate grant", VersionOfApi) { + val (code, body) = createSystemEntity(entityRowLevel) + code should equal(201) + val dynamicEntityId = (body \ "dynamic_entity_id").extract[String] + try { + val id = createRecord("shared record") + + When("user1 (owner, holds ACL CanGrant — no role needed) grants READ to user2") + val grantResp = makePutRequest( + (dynamicEntity_Request / "test_rl" / id / "access").PUT <@ (user1), + write(("user_id" -> resourceUser2.userId) ~ ("can_read" -> true) ~ ("can_update" -> false))) + grantResp.code should equal(200) + + Then("user2 can now read the record") + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user2)).code should equal(200) + + When("user2 (read-only) tries to update the record") + Then("it is 403 — read does not confer update") + makePutRequest((dynamicEntity_Request / "test_rl" / id).PUT <@ (user2), write(("name" -> "u2 edit"): JValue)) + .code should equal(403) + + When("user1 additionally grants UPDATE to user2") + makePutRequest( + (dynamicEntity_Request / "test_rl" / id / "access").PUT <@ (user1), + write(("user_id" -> resourceUser2.userId) ~ ("can_read" -> true) ~ ("can_update" -> true))) + .code should equal(200) + + Then("user2 can now update") + makePutRequest((dynamicEntity_Request / "test_rl" / id).PUT <@ (user2), write(("name" -> "u2 edit"): JValue)) + .code should equal(200) + } finally deleteSystemEntity(dynamicEntityId) + } + } + + // ==================== Feature 3: revoke ==================== + + feature("Feature 3: revoke removes access") { + scenario("3.1: revoke a grantee → they lose read", VersionOfApi) { + val (code, body) = createSystemEntity(entityRowLevel) + code should equal(201) + val dynamicEntityId = (body \ "dynamic_entity_id").extract[String] + try { + val id = createRecord("revoke me") + makePutRequest( + (dynamicEntity_Request / "test_rl" / id / "access").PUT <@ (user1), + write(("user_id" -> resourceUser2.userId) ~ ("can_read" -> true))).code should equal(200) + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user2)).code should equal(200) + + When("user1 revokes user2") + val revoke = makeDeleteRequest((dynamicEntity_Request / "test_rl" / id / "access" / resourceUser2.userId).DELETE <@ (user1)) + revoke.code should equal(200) + (revoke.body \ "revoked_count").extract[Int] should be >= 1 + + Then("user2 can no longer read the record") + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user2)).code should equal(404) + } finally deleteSystemEntity(dynamicEntityId) + } + } + + // ==================== Feature 4: admin-role override ==================== + + feature("Feature 4: CanGrantDynamicEntityRowAccess admin override") { + scenario("4.1: a role holder may list/grant access on a row they cannot read", VersionOfApi) { + val (code, body) = createSystemEntity(entityRowLevel) + code should equal(201) + val dynamicEntityId = (body \ "dynamic_entity_id").extract[String] + try { + val id = createRecord("admin-managed") + + When("user3 (no ACL, no role) lists access on the row") + Then("it is 403") + makeGetRequest((dynamicEntity_Request / "test_rl" / id / "access").GET <@ (user3)).code should equal(403) + + When("user3 is granted the admin row-access role") + Entitlement.entitlement.vend.addEntitlement("", resourceUser3.userId, "CanGrantDynamicEntityRowAccess_Systemtest_rl") + + Then("user3 can list access even though it cannot read the row") + makeGetRequest((dynamicEntity_Request / "test_rl" / id / "access").GET <@ (user3)).code should equal(200) + + And("user3 can grant access to user2") + makePutRequest( + (dynamicEntity_Request / "test_rl" / id / "access").PUT <@ (user3), + write(("user_id" -> resourceUser2.userId) ~ ("can_read" -> true))).code should equal(200) + makeGetRequest((dynamicEntity_Request / "test_rl" / id).GET <@ (user2)).code should equal(200) + } finally deleteSystemEntity(dynamicEntityId) + } + } + + // ==================== Feature 5: flag-off & definition-time validation ==================== + + feature("Feature 5: flag-off and mutual-exclusion validation") { + scenario("5.1: access endpoints return 400 for a non-row-level entity", VersionOfApi) { + val normalEntity: JValue = + ("entity_name" -> "test_normal_rl") ~ ("has_personal_entity" -> false) ~ ("schema" -> simpleSchema) + val (code, body) = createSystemEntity(normalEntity) + code should equal(201) + val dynamicEntityId = (body \ "dynamic_entity_id").extract[String] + try { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, "CanCreateDynamicEntity_Systemtest_normal_rl") + val createResp = makePostRequest((dynamicEntity_Request / "test_normal_rl").POST <@ (user1), write(("name" -> "x"): JValue)) + createResp.code should equal(201) + val id = (createResp.body \ "test_normal_rl" \ "test_normal_rl_id").extract[String] + + When("we hit the row-access endpoint on an entity without use_row_level_access") + Then("it is 400") + makeGetRequest((dynamicEntity_Request / "test_normal_rl" / id / "access").GET <@ (user1)).code should equal(400) + } finally deleteSystemEntity(dynamicEntityId) + } + + scenario("5.2: use_row_level_access cannot be combined with has_public_access (§8.3)", VersionOfApi) { + val contradictory: JValue = + ("entity_name" -> "test_rl_conflict") ~ + ("use_row_level_access" -> true) ~ + ("has_public_access" -> true) ~ + ("schema" -> simpleSchema) + When("we create an entity with both flags") + val (code, _) = createSystemEntity(contradictory) + Then("it is rejected (not 201)") + code should equal(400) + } + } + +} diff --git a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala index 2bb7066acf..bb4f194aae 100644 --- a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala +++ b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala @@ -272,6 +272,14 @@ object DynamicEntityFieldType extends OBPEnumeration[DynamicEntityFieldType]{ } override def wrongTypeMsg: String = "the value's type should be a JSON object or array." } + // A field declared `reference:` (validated elsewhere via the "reference:" prefix). Its stored + // value is an id String, so it behaves like `string` for indexing/projection/query. This enum value is + // the internal type the query layer carries for such fields; the concrete target entity is tracked + // separately (DynamicEntityInfo.referenceFields). It is deliberately excluded from the declarable + // field-type allow-list (DynamicEntityProvider.allowedFieldType) — callers still declare `reference:X`. + object reference extends Value { + val jValueType = classOf[JString] + } //object array extends Value{val jValueType = classOf[JArray]} //object `object` extends Value{val jValueType = classOf[JObject]} //TODO in the future, we consider support nested type }