Fixes from Sentinel 2

obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200AIS.scala

@@ -189,7 +189,8 @@ object Http4sUKOBv200AIS extends MdcLoggable {
           (transactions, _) <- Future {
             BankAccountExtended(account).getModeratedTransactions(bank, Full(u), view, BankIdAccountId(account.bankId, account.accountId), Some(cc), params)
           } map { x => unboxFull(fullBoxOrException(x ~> APIFailureNewStyle(UnknownError, 400, Some(cc.toLight)))) }
-        } yield JSONFactory_UKOpenBanking_200.createTransactionsJson(transactions, Nil)
+          // The account id comes from the account looked up above, not from the first transaction, which an empty result does not have.
+        } yield JSONFactory_UKOpenBanking_200.createTransactionsJson(account.accountId.value, transactions, Nil)
       }
   }
   resourceDocs += ResourceDoc(

obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/JSONFactory_UKOpenBanking_200.scala

@@ -170,8 +170,8 @@ object JSONFactory_UKOpenBanking_200 extends CustomJsonFormats {
     )
   }
 
-  def createTransactionsJson(transactions: List[ModeratedTransaction], transactionRequests: List[TransactionRequest]) : TransactionsJsonUKV200 = {
-    val accountId = transactions.head.bankAccount.get.accountId.value
+  // The caller passes the account id from the path, so an account with no transactions gives an empty list, not an exception.
+  def createTransactionsJson(accountId: String, transactions: List[ModeratedTransaction], transactionRequests: List[TransactionRequest]) : TransactionsJsonUKV200 = {
     val transactionsInnerJson = transactions.map(
       transaction=>TransactionInnerJson(
         AccountId = accountId,

obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13AIS.scala

@@ -265,7 +265,9 @@ object Http4sBGv13AIS extends MdcLoggable {
         val u = cc.user.openOrThrowException(AuthenticatedUserIsRequired)
         for {
           _ <- passesPsd2Aisp(callContext)
-          availablePrivateAccounts <- Views.views.vend.getPrivateBankAccountsFuture(u)
+          // only accounts the user reaches through the ReadAccountsBerlinGroup view (403 if none), as getAccountList
+          // and readCardAccount do, else any private view (owner, accountant, ...) would list a card account
+          (availablePrivateAccounts, callContext) <- NewStyle.function.getAccountListOfBerlinGroup(u, callContext)
           (_, callContext) <- NewStyle.function.getPhysicalCardsForUser(u, callContext)
           (accounts, callContext) <- NewStyle.function.getBankAccounts(availablePrivateAccounts, callContext)
           (canReadBalancesAccounts, callContext) <- NewStyle.function.getAccountCanReadBalancesOfBerlinGroup(u, callContext)
@@ -422,6 +424,9 @@ object Http4sBGv13AIS extends MdcLoggable {
         val callContext = Some(cc)
         val user = cc.user.openOrThrowException(AuthenticatedUserIsRequired)
         for {
+          // The calling TPP must hold a PSD2 certificate with the PSP_AI role, as for the transaction
+          // list of the same account; else a consumer without it could still read single transactions.
+          _ <- passesPsd2Aisp(callContext)
           (account: BankAccount, callContext) <- NewStyle.function.getBankAccountByAccountId(AccountId(accountId), callContext)
           viewId = ViewId(SYSTEM_READ_TRANSACTIONS_BERLIN_GROUP_VIEW_ID)
           bankIdAccountId = BankIdAccountId(account.bankId, account.accountId)
@@ -1245,7 +1250,8 @@ respectively the OAuth2 access token.
     }
   ]
 }""")),
-      List(AuthenticatedUserIsRequired, UnknownError),
+      // 403 NoViewReadAccountsBerlinGroup when the user has no account through the ReadAccountsBerlinGroup view
+      List(AuthenticatedUserIsRequired, NoViewReadAccountsBerlinGroup, UnknownError),
       ApiTag("Account Information Service (AIS)") :: apiTagMockedData :: Nil,
       http4sPartialFunction = Some(getCardAccounts)
     )
@@ -1396,7 +1402,8 @@ of the "Read Transaction List" call within the _links subfield.
     }
   }
 }""")),
-      List(AuthenticatedUserIsRequired, UnknownError),
+      // The X509 / regulated-entity errors come from passesPsd2Aisp, when PSD2 certificates are required
+      List(AuthenticatedUserIsRequired, X509CannotGetCertificate, RegulatedEntityNotFoundByCertificate, X509ActionIsNotAllowed, X509GeneralError, UnknownError),
       ApiTag("Account Information Service (AIS)") :: Nil,
       http4sPartialFunction = Some(getTransactionDetails)
     )

obp-api/src/main/scala/code/api/cache/Redis.scala

@@ -102,6 +102,46 @@
     jedis
   }
 
+  private val SubscriberMinBackoffMs = 1000L  // first wait after a lost subscriber connection
+  private val SubscriberMaxBackoffMs = 30000L // longest wait between reconnect attempts while Redis is down
+
+  /**
+   * Run an event bus's blocking subscribe on a dedicated connection for as long as `running` is true.
+   *
+   * When the connection fails (e.g. Redis restarts or drops the socket) the subscribe call throws;
+   * this closes that connection, waits (1s, doubling up to 30s) and subscribes again on a new one,
+   * so the bus's gRPC streams get messages again once Redis is back, without restarting OBP-API.
+   * `onConnection` hands each new connection to the bus, so its stop() can close the current one.
+   */
+  def runSubscriber(busName: String, running: => Boolean, onConnection: Jedis => Unit)(subscribe: Jedis => Unit): Unit = {
+    var backoffMs = SubscriberMinBackoffMs // wait before the next attempt; doubles per failure, capped
+    var failed = false // set after a failure, so the next connection is logged as a reconnect
+    while (running) { // keep subscribing until stop() clears running
+      var jedis: Jedis = null
+      val startedAt = System.currentTimeMillis() // how long this subscription lasted decides the next backoff
+      try {
+        jedis = newSubscriberConnection()
+        onConnection(jedis)
+        if (failed) logger.info(s"$busName says: Redis subscriber reconnected") // shows streaming resumed after an outage
+        subscribe(jedis) // blocks until unsubscribed by stop() or the connection fails
+      } catch {
+        case e: Throwable if running =>
+          // a subscription that lasted a while means Redis was up: start again from the shortest wait
+          if (System.currentTimeMillis() - startedAt >= SubscriberMaxBackoffMs) backoffMs = SubscriberMinBackoffMs
+          // the connection is gone but the bus is still meant to run: log and retry below
+          logger.error(s"$busName says: Redis subscriber connection lost, reconnecting in ${backoffMs}ms: ${e.getMessage}")
+          failed = true
+        case _: Throwable => // shutting down, ignore
+      } finally {
+        try { if (jedis != null) jedis.close() } catch { case _: Throwable => } // release the dead socket before opening a new one
+      }
+      if (running) { // wait before retrying, so a Redis outage is not hammered with reconnects
+        try Thread.sleep(backoffMs) catch { case _: InterruptedException => return } // interrupted: the JVM is shutting down
+        backoffMs = math.min(backoffMs * 2, SubscriberMaxBackoffMs) // wait longer each time Redis is still down
+      }
+    }
+  }
+
   // Redis startup health check
   private def performStartupHealthCheck(): Unit = {
     try {

obp-api/src/main/scala/code/api/util/Glossary.scala

@@ -1808,6 +1808,63 @@ object Glossary extends MdcLoggable  {
 				|<img width="468" alt="OBP Access Control Image" src="$getServerUrl/media/images/glossary/OBP_Consent_Request__3_.png"></img>
 				|""".stripMargin)
 
+	glossaryItems += GlossaryItem(
+		title = "On Behalf Of",
+		description = onBehalfOfDescription
+	)
+
+	/**
+	 * This builds the "On Behalf Of" description. The table is rendered from UserReference, which
+	 * holds the policy of every column that stores a user id and which of them are applied yet, so
+	 * the Glossary cannot drift from the code. The table is appended after stripMargin, because its
+	 * rows start with "|" and stripMargin would eat that.
+	 */
+	private def onBehalfOfDescription: String = {
+		import code.users.{AttributionPolicy, UserReference}
+		import code.users.UserReference.NotYetWired
+
+		def policyText(policy: AttributionPolicy): String = policy match {
+			case AttributionPolicy.UseAuthenticatedUserId => "the caller"
+			case AttributionPolicy.UseOnBehalfOfUserId    => "the User acted for"
+			case AttributionPolicy.Reject                 => "refused"
+		}
+		def todayText(ref: UserReference): String = UserReference.notYetWired.get(ref) match {
+			case None                                                                => policyText(ref.policy)
+			case Some(NotYetWired.StoresCaller(_)) if ref.policy == AttributionPolicy.Reject => "the caller (not refused)"
+			case Some(NotYetWired.StoresCaller(_))                                   => "the caller"
+			case Some(NotYetWired.NotReachedByConsentUser(_))                        => "not written by a consent user"
+		}
+		def cell(text: String): String = text.replace("|", "\\|")
+
+		val rows = UserReference.all.map { ref =>
+			s"| ${ref.name.takeWhile(_ != '_')} | ${ref.fields.map(field => s"`$field`").mkString(", ")} | " +
+			s"${policyText(ref.policy)} | ${todayText(ref)} | ${cell(ref.note)} |"
+		}
+		val delegated = UserReference.all.filter(ref => ref.policy != AttributionPolicy.UseAuthenticatedUserId)
+		val applied = delegated.count(UserReference.isWired)
+
+		s"""When a User grants a ${getGlossaryItemLink("Consent")} to an application, such as an AI agent, the application calls OBP under an identity of its own: a consent user, which OBP creates for that Consent. A call made that way has two Users behind it. `user_id` is the caller, the consent user that authenticated. `on_behalf_of_user_id` is the User the call was made for, the one who granted the Consent. When a User calls OBP for themselves, the two are the same User.
+			 |
+			 |OBP finds the User acted for itself, from the Consent the caller authenticated with. It never takes it from a request body, header or query parameter, so a caller cannot name somebody else to act for.
+			 |
+			 |This matters whenever OBP stores a user id. A consent user belongs to its Consent, so a row stored against it, such as an account holder, belongs to an identity that ends with the Consent, and the User who asked for it can no longer see it. So every database column that stores a user id has a policy, which says whose id to store when a consent user makes the call:
+			 |
+			 |* **the caller**: store the consent user's own id. This suits audit records of who acted, and the rows that carry a Consent's own permissions, which are meant to end with it.
+			 |* **the User acted for**: store the id of the User who granted the Consent, so the row outlives the Consent. A few tables keep both ids, for example a counterparty or a transaction request, where who made the call is worth keeping too; the Note column says so.
+			 |* **refused**: a consent user must not do this at all, because what it would create (a Consent, a Consumer or a token) outlives the Consent and would let the delegation extend itself.
+			 |
+			 |A request that explicitly names a consent user as the target User, for example in a `user_id` field or a `USER_ID` path segment, is answered with 400 `InvalidUserId`.
+			 |
+			 |Where a v7.0.0 response publishes both ids, it names them `user_id` and `on_behalf_of_user_id`, as in the record metadata of a ${getGlossaryItemLink("Dynamic-Entities")} record.
+			 |
+			 |**Every column that stores a user id**
+			 |
+			 |Policy is whose id the column should hold when a consent user makes the call. Today is whose id it holds on this instance now. Of the ${delegated.size} columns whose policy is "the User acted for" or "refused", ${applied} apply it today. "Not written by a consent user" means no API call writes that column with a consent user's id, because the endpoint refuses the consent user first or no endpoint writes the table.
+			 |
+			 |""".stripMargin +
+		(List("| Record | Column | Policy | Today | Note |", "|---|---|---|---|---|") ++ rows).mkString("\n") + "\n"
+	}
+
 
 	glossaryItems += GlossaryItem(
 		title = "Authentication: Consent OBP Flow Example",

obp-api/src/main/scala/code/api/util/NewStyle.scala

@@ -5103,7 +5103,8 @@ object NewStyle extends MdcLoggable{
       callContext: Option[CallContext]
     ): OBPReturnType[com.openbankproject.commons.model.MarketOrder] = {
       Connector.connector.vend.getMarketOrder(bankId, accountId, orderId, callContext) map {
-        i => (unboxFullOrFail(i._1, callContext, s"$OrderNotFound"), i._2)
+        // 404: a missing order and another account's order look the same
+        i => (unboxFullOrFail(i._1, callContext, s"$OrderNotFound", 404), i._2)
       }
     }
 
@@ -5114,7 +5115,8 @@ object NewStyle extends MdcLoggable{
       callContext: Option[CallContext]
     ): OBPReturnType[com.openbankproject.commons.model.MarketOrder] = {
       Connector.connector.vend.cancelMarketOrder(bankId, accountId, orderId, callContext) map {
-        i => (unboxFullOrFail(i._1, callContext, s"$OrderNotFound"), i._2)
+        // 404: a missing order and another account's order look the same
+        i => (unboxFullOrFail(i._1, callContext, s"$OrderNotFound", 404), i._2)
       }
     }
 
@@ -5141,7 +5143,8 @@ object NewStyle extends MdcLoggable{
       callContext: Option[CallContext]
     ): OBPReturnType[com.openbankproject.commons.model.MarketTrade] = {
       Connector.connector.vend.getMarketTrade(bankId, accountId, tradeId, callContext) map {
-        i => (unboxFullOrFail(i._1, callContext, s"$TradeNotFound"), i._2)
+        // 404: a missing trade and another account's trade look the same
+        i => (unboxFullOrFail(i._1, callContext, s"$TradeNotFound", 404), i._2)
       }
     }
 

obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala

@@ -673,6 +673,13 @@ object Http4s121 {
               s"${CreateCustomViewError} You need the `${CAN_UPDATE_CUSTOM_VIEW}` permission on any your views",
               cc = Some(cc)
             )(anyViewContainsPermission)
+            // the view may only get actions the caller already has on one of their views of this account, else they grant themselves more (as v5.1.0 does)
+            callerActions = Views.views.vend.permission(BankIdAccountId(account.bankId, account.accountId), user)
+              .map(_.views.flatMap(_.allowed_actions)).getOrElse(Nil).toSet
+            _ <- Helper.booleanToFuture(
+              s"$SourceViewHasLessPermission Current permissions of your views ($callerActions), target view permissions (${updateJsonV121.allowed_actions})",
+              failCode = 403, cc = Some(cc)
+            )(updateJsonV121.allowed_actions.toSet.subsetOf(callerActions))
             updatedView <- Future(Views.views.vend.updateCustomView(BankIdAccountId(account.bankId, account.accountId), ViewId(viewId), updateViewJson)) map {
               unboxFullOrFail(_, Some(cc), CreateCustomViewError)
             }
@@ -694,7 +701,8 @@ object Http4s121 {
       |of a view is not editable (it is only set when a view is created)""",
       updateViewJsonV121,
       viewJSONV121,
-      List(InvalidJsonFormat, AuthenticatedUserIsRequired, BankAccountNotFound, ViewNotFound, UnknownError, "user does not have owner access"),
+      // SourceViewHasLessPermission: the body asks for actions none of the caller's views on the account has
+      List(InvalidJsonFormat, AuthenticatedUserIsRequired, BankAccountNotFound, ViewNotFound, SourceViewHasLessPermission, UnknownError, "user does not have owner access"),
       List(apiTagAccount, apiTagView, apiTagOldStyle),
       http4sPartialFunction = Some(updateViewForBankAccount)
     )

obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala

@@ -168,7 +168,11 @@ object Http4s140 {
       case req @ POST -> `prefixPath` / "banks" / _ / "customer" / customerId / "messages" =>
         EndpointHelpers.withUserAndBankAndBodyCreated[JSONFactory1_4_0.AddCustomerMessageJson, SuccessMessage](req) { (_, bank, body, cc) =>
           for {
-            (_, cc2)          <- NewStyle.function.getCustomerByCustomerId(customerId, Some(cc))
+            (customer, cc2)   <- NewStyle.function.getCustomerByCustomerId(customerId, Some(cc))
+            // the customer must be at the bank in the path, else a caller at one bank messages another bank's customers
+            _                 <- code.util.Helper.booleanToFuture(CustomerNotFoundByCustomerId, 404, Some(cc)) {
+              customer.bankId == bank.bankId.value
+            }
             (ucl, cc3)        <- NewStyle.function.getUserCustomerLinkByCustomerId(customerId, cc2)
             (targetUser, cc4) <- NewStyle.function.findByUserId(ucl.userId, cc3)
             (_, _)            <- NewStyle.function.createMessage(targetUser, bank.bankId, body.message, body.from_department, body.from_person, cc4)
@@ -185,9 +189,11 @@ object Http4s140 {
       "Create a message for the customer specified by CUSTOMER_ID",
       addCustomerMessageJson,
       successMessage,
-      List(AuthenticatedUserIsRequired, UnknownError),
+      // the role, customer and user-link errors this endpoint can return
+      List(AuthenticatedUserIsRequired, UserHasMissingRoles, CustomerNotFoundByCustomerId, UserCustomerLinksNotFoundForUser, UnknownError),
       List(apiTagMessage, apiTagCustomer, apiTagPerson),
-      None,
+      // the caller needs CanCreateCustomerMessage, as in v4.0.0, else any logged-in user writes to any customer's inbox
+      Some(List(canCreateCustomerMessage)),
       http4sPartialFunction = Some(addCustomerMessage)
     )
 

obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala

@@ -497,6 +497,10 @@ object Http4s200 {
               APIUtil.hasEntitlement(bank.bankId.value, user.userId, canGetSocialMediaHandles)
             }
             (customer, _) <- NewStyle.function.getCustomerByCustomerId(customerId, Some(cc))
+            // the customer must be at the bank in the path, else a Role at one bank reads another bank's customers' handles
+            _ <- code.util.Helper.booleanToFuture(CustomerNotFoundByCustomerId, 404, Some(cc)) {
+              customer.bankId == bank.bankId.value
+            }
           } yield {
             val socialMedias = SocialMediaHandle.socialMediaHandleProvider.vend.getSocialMedias(customer.number)
             createSocialMediasJSON(socialMedias)
@@ -644,10 +648,15 @@ object Http4s200 {
             _ <- code.util.Helper.booleanToFuture(UserHasMissingRoles + canAddSocialMediaHandle, failCode = 403, cc = Some(cc)) {
               APIUtil.hasEntitlement(bank.bankId.value, user.userId, canAddSocialMediaHandle)
             }
-            (_, _) <- NewStyle.function.getCustomerByCustomerId(customerId, Some(cc))
+            (customer, _) <- NewStyle.function.getCustomerByCustomerId(customerId, Some(cc))
+            // the customer must be at the bank in the path, else the Role at one bank writes to any bank's customer
+            _ <- code.util.Helper.booleanToFuture(CustomerNotFoundByCustomerId, 404, Some(cc)) {
+              customer.bankId == bank.bankId.value
+            }
             _ <- code.util.Helper.booleanToFuture("Server error: could not add", cc = Some(cc)) {
               SocialMediaHandle.socialMediaHandleProvider.vend.addSocialMedias(
-                body.customer_number, body.`type`, body.handle,
+                // stored against the number of the customer in the path, not the body's customer_number, so it cannot target another customer
+                customer.number, body.`type`, body.handle,
                 body.date_added, body.date_activated)
             }
           } yield SuccessMessage("Success")