diff --git a/CLAUDE.md b/CLAUDE.md index d1ffcd6a..633b8fb8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,7 +190,10 @@ MistKit/ | `CloudKitService+ZoneOperations.swift` | `listZones`, `lookupZones(zoneIDs:)`, `fetchZoneChanges(syncToken:)` | | `CloudKitService+ModifyZones.swift` | `modifyZones(_:database:)` | | `CloudKitService+SyncOperations.swift` | `fetchRecordChanges(recordType:syncToken:)`, `fetchAllRecordChanges(recordType:syncToken:)` | -| `CloudKitService+UserOperations.swift` | `fetchCaller()`, `discoverUserIdentities(lookupInfos:)`, `discoverAllUserIdentities()` *(unavailable — pending #28)*, `lookupUsersByEmail(_:)`, `lookupUsersByRecordName(_:)`, `fetchCurrentUser()` (deprecated, forwards to `fetchCaller`) | +| `CloudKitService+UserOperations.swift` | `fetchCaller()`, `discoverUserIdentities(lookupInfos:)`, `discoverAllUserIdentities()` *(no-arg address-book form — unavailable, pending #28; distinct from the available `discoverAllUserIdentities(lookupInfos:batchSize:)` chunking overload below)*, `lookupUsersByEmail(_:)`, `lookupUsersByRecordName(_:)`, `fetchCurrentUser()` (deprecated, forwards to `fetchCaller`) | +| `CloudKitService+LookupAllRecords.swift` | `lookupAllRecords(recordNames:desiredKeys:database:batchSize:)` — auto-chunking convenience over `lookupRecords` | +| `CloudKitService+UserIdentityChunking.swift` | `discoverAllUserIdentities(lookupInfos:batchSize:)` — auto-chunking convenience over `discoverUserIdentities` | +| `CloudKitService+BatchChunking.swift` | internal `chunkedBatches` helper backing the auto-chunking conveniences | | `CloudKitService+AssetOperations.swift` | `uploadAssets`, `requestAssetUploadURL` | | `CloudKitService+AssetUpload.swift` | `uploadAssetData` | | `CloudKitService+RecordManaging.swift` | record-managing convenience surface | @@ -210,6 +213,17 @@ MistKit/ - `lookupUsersByEmail(_:)` → POST `/users/lookup/email` — returns `[UserIdentity]`. - `lookupUsersByRecordName(_:)` → POST `/users/lookup/id` — returns `[UserIdentity]`. +**Batch chunking (issue #307):** the two non-deprecated operations capped at CloudKit's 200-item-per-request limit (`CloudKitService.maxRecordsPerRequest`) each pair a single-request primitive with an auto-chunking convenience that splits the input into ≤`batchSize` batches, calls the primitive per batch, and concatenates results in input order. This mirrors the `queryRecords`/`queryAllRecords` page-primitive + auto-paginating-extension pattern. Because chunk count is `ceil(input.count / batchSize)` — deterministic and finite — there is **no** `maxPages`-style throwing ceiling; `batchSize` (default `maxRecordsPerRequest`, clamped to `1...maxRecordsPerRequest`) is the only knob. The shared engine is `chunkedBatches` (`CloudKitService+BatchChunking.swift`). + +| Primitive (single request) | Auto-chunking convenience | +|----------------------------|---------------------------| +| `lookupRecords(recordNames:desiredKeys:database:)` | `lookupAllRecords(recordNames:desiredKeys:database:batchSize:)` | +| `discoverUserIdentities(lookupInfos:)` | `discoverAllUserIdentities(lookupInfos:batchSize:)` *(overloads the no-arg address-book form)* | + +The `users/lookup/email` and `users/lookup/id` primitives (`lookupUsersByEmail` / `lookupUsersByRecordName`) are **deprecated by Apple** in favor of POST `users/discover` (verified against Apple's archived CloudKit Web Services reference), so they intentionally get **no** chunking convenience — callers needing >200 should use `discoverAllUserIdentities(lookupInfos:)`. `users/lookup/contacts` is likewise deprecated and unwrapped. + +`listZones` is **not** a pagination candidate — `zones/list` (GET) returns every zone in one response with no continuation marker. `modifyRecords`/`sync` already chunk by 200 internally. The `fetchAllRecordChanges` / `fetchAllZoneChanges` paginators already implement the page-primitive pattern with `maxPages` + stuck-token detection. + In MistDemo, integration runs targeting these endpoints use `PhaseContext.userContextService` (a public+web-auth `CloudKitService`) which is built from `CLOUDKIT_API_TOKEN` + `CLOUDKIT_WEB_AUTH_TOKEN` regardless of the primary `--database` selection. The `DatabaseConfiguration` / `AuthenticationCredentials` types in `Examples/MistDemo/Sources/MistDemoKit/Configuration/` enforce valid database+auth combinations at construction time. **Result Types (Sources/MistKit/Models/ and Sources/MistKit/Models/Zones/):** diff --git a/Examples/MistDemo/Sources/MistDemoKit/Commands/DiscoverAllUserIdentitiesCommand.swift b/Examples/MistDemo/Sources/MistDemoKit/Commands/DiscoverAllUserIdentitiesCommand.swift new file mode 100644 index 00000000..5347b3c1 --- /dev/null +++ b/Examples/MistDemo/Sources/MistDemoKit/Commands/DiscoverAllUserIdentitiesCommand.swift @@ -0,0 +1,102 @@ +// +// DiscoverAllUserIdentitiesCommand.swift +// MistDemo +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +internal import Foundation +internal import MistKit + +/// Command that discovers user identities for a set of email addresses using +/// the auto-chunking `discoverAllUserIdentities(lookupInfos:)` convenience +/// (issue #307). +public struct DiscoverAllUserIdentitiesCommand: MistDemoCommand, OutputFormatting { + /// The configuration type. + public typealias Config = DiscoverConfig + /// The command name. + public static let commandName = "discover-all" + /// The command abstract. + public static let abstract = + "Discover user identities, auto-chunking large inputs (discoverAllUserIdentities)" + /// The command help text. + public static let helpText = """ + DISCOVER-ALL - Discover user identities, auto-chunking past CloudKit's 200/request cap + + USAGE: + mistdemo discover-all --discover-emails [options] + + INPUT (choose one): + --discover-emails Comma-separated email addresses + --stdin Read one email per line from stdin + + OPTIONS: + --batch-size Items per request (default 200, clamped 1...200). + Set small (e.g. 1) to force multiple requests. + --output-format Output format (json, table, csv, yaml) + + NOTES: + - Requires API + web-auth credentials; the endpoint is pinned to the + public database, so the --database flag does not apply. + - Each email is sent as a lookup info; CloudKit only returns identities + for accounts discoverable to the caller. + """ + + private let config: DiscoverConfig + + /// Creates a new instance. + public init(config: DiscoverConfig) { + self.config = config + } + + /// Executes the command. + public func execute() async throws { + guard !config.emails.isEmpty else { + throw DiscoverError.emailsRequired + } + guard config.base.hasUserContextCredentials else { + throw DiscoverError.webAuthRequired + } + + let service = try MistKitClientFactory.create(for: config.base) + + let effectiveBatchSize = min( + max(config.batchSize, 1), + CloudKitService.maxRecordsPerRequest + ) + let batches = (config.emails.count + effectiveBatchSize - 1) / effectiveBatchSize + let note = + "discover-all: \(config.emails.count) lookup(s), batchSize \(config.batchSize) " + + "→ \(batches) request(s)\n" + FileHandle.standardError.write(Data(note.utf8)) + + let lookupInfos = config.emails.map { UserIdentityLookupInfo(emailAddress: $0) } + let identities = try await service.discoverAllUserIdentities( + lookupInfos: lookupInfos, + batchSize: config.batchSize + ) + try await outputResults(identities, format: config.output) + } +} diff --git a/Examples/MistDemo/Sources/MistDemoKit/Commands/LookupAllRecordsCommand.swift b/Examples/MistDemo/Sources/MistDemoKit/Commands/LookupAllRecordsCommand.swift new file mode 100644 index 00000000..8b45bb9a --- /dev/null +++ b/Examples/MistDemo/Sources/MistDemoKit/Commands/LookupAllRecordsCommand.swift @@ -0,0 +1,99 @@ +// +// LookupAllRecordsCommand.swift +// MistDemo +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +internal import Foundation +internal import MistKit + +/// Command to look up records by name using the auto-chunking +/// `lookupAllRecords` convenience (issue #307). +public struct LookupAllRecordsCommand: MistDemoCommand, OutputFormatting { + /// The configuration type. + public typealias Config = LookupConfig + /// The command name. + public static let commandName = "lookup-all" + /// The command abstract. + public static let abstract = + "Look up records by name, auto-chunking large inputs (lookupAllRecords)" + /// The command help text. + public static let helpText = """ + LOOKUP-ALL - Fetch records by name, auto-chunking past CloudKit's 200/request cap + + USAGE: + mistdemo lookup-all --record-names [options] + + REQUIRED: + --api-token CloudKit API token + --web-auth-token Web authentication token + --record-names Comma-separated record names + + OPTIONS: + --fields Restrict returned fields + --batch-size Items per request (default 200, clamped 1...200). + Set small (e.g. 1) to force multiple requests. + --output-format Output format (json, table, csv, yaml) + + EXAMPLES: + mistdemo lookup-all --record-names note-1,note-2,note-3 --batch-size 1 + """ + + private let config: LookupConfig + + /// Creates a new instance. + public init(config: LookupConfig) { + self.config = config + } + + /// Executes the command. + public func execute() async throws { + let client = try MistKitClientFactory.create(for: config.base) + + let effectiveBatchSize = min( + max(config.batchSize, 1), + CloudKitService.maxRecordsPerRequest + ) + let batches = + (config.recordNames.count + effectiveBatchSize - 1) / effectiveBatchSize + let note = + "lookup-all: \(config.recordNames.count) name(s), batchSize \(config.batchSize) " + + "→ \(batches) request(s)\n" + FileHandle.standardError.write(Data(note.utf8)) + + let results = try await client.lookupAllRecords( + recordNames: config.recordNames, + desiredKeys: config.fields, + database: config.base.database, + batchSize: config.batchSize + ) + + let records = results.compactMap { result in + if case .success(let record) = result { record } else { nil } + } + try await outputResults(records, format: config.output) + } +} diff --git a/Examples/MistDemo/Sources/MistDemoKit/Configuration/DiscoverConfig.swift b/Examples/MistDemo/Sources/MistDemoKit/Configuration/DiscoverConfig.swift index 825e2bde..f48fb494 100644 --- a/Examples/MistDemo/Sources/MistDemoKit/Configuration/DiscoverConfig.swift +++ b/Examples/MistDemo/Sources/MistDemoKit/Configuration/DiscoverConfig.swift @@ -29,6 +29,7 @@ public import ConfigKeyKit internal import Foundation +public import MistKit /// Configuration for the `discover` command (email lookup). public struct DiscoverConfig: Sendable, ConfigurationParseable { @@ -41,6 +42,9 @@ public struct DiscoverConfig: Sendable, ConfigurationParseable { public let base: MistDemoConfig /// The email addresses to look up. public let emails: [String] + /// Maximum items per request for the auto-chunking `discover-all` command + /// (the plain `discover` command ignores it). + public let batchSize: Int /// The output format. public let output: OutputFormat @@ -48,10 +52,12 @@ public struct DiscoverConfig: Sendable, ConfigurationParseable { public init( base: MistDemoConfig, emails: [String], + batchSize: Int = CloudKitService.maxRecordsPerRequest, output: OutputFormat = .json ) { self.base = base self.emails = emails + self.batchSize = batchSize self.output = output } @@ -79,9 +85,16 @@ public struct DiscoverConfig: Sendable, ConfigurationParseable { ) ?? MistDemoConstants.Defaults.outputFormat let output = OutputFormat(rawValue: outputString) ?? .json + let batchSize = + configuration.int( + forKey: MistDemoConstants.ConfigKeys.batchSize, + default: CloudKitService.maxRecordsPerRequest + ) ?? CloudKitService.maxRecordsPerRequest + self.init( base: baseConfig, emails: emails, + batchSize: batchSize, output: output ) } diff --git a/Examples/MistDemo/Sources/MistDemoKit/Configuration/LookupConfig.swift b/Examples/MistDemo/Sources/MistDemoKit/Configuration/LookupConfig.swift index f03cacae..edc14289 100644 --- a/Examples/MistDemo/Sources/MistDemoKit/Configuration/LookupConfig.swift +++ b/Examples/MistDemo/Sources/MistDemoKit/Configuration/LookupConfig.swift @@ -29,6 +29,7 @@ public import ConfigKeyKit internal import Foundation +public import MistKit /// Configuration for lookup command. public struct LookupConfig: Sendable, ConfigurationParseable { @@ -43,6 +44,9 @@ public struct LookupConfig: Sendable, ConfigurationParseable { public let recordNames: [String] /// The optional field names to include in the response. public let fields: [String]? + /// Maximum items per request for the auto-chunking `lookup-all` command + /// (the plain `lookup` command ignores it). + public let batchSize: Int /// The output format. public let output: OutputFormat @@ -51,11 +55,13 @@ public struct LookupConfig: Sendable, ConfigurationParseable { base: MistDemoConfig, recordNames: [String], fields: [String]? = nil, + batchSize: Int = CloudKitService.maxRecordsPerRequest, output: OutputFormat = .json ) { self.base = base self.recordNames = recordNames self.fields = fields + self.batchSize = batchSize self.output = output } @@ -111,10 +117,17 @@ public struct LookupConfig: Sendable, ConfigurationParseable { ) ?? MistDemoConstants.Defaults.outputFormat let output = OutputFormat(rawValue: outputString) ?? .json + let batchSize = + configReader.int( + forKey: MistDemoConstants.ConfigKeys.batchSize, + default: CloudKitService.maxRecordsPerRequest + ) ?? CloudKitService.maxRecordsPerRequest + self.init( base: baseConfig, recordNames: recordNames, fields: fields, + batchSize: batchSize, output: output ) } diff --git a/Examples/MistDemo/Sources/MistDemoKit/Constants/MistDemoConstants.swift b/Examples/MistDemo/Sources/MistDemoKit/Constants/MistDemoConstants.swift index 454c0aff..a5c2d40d 100644 --- a/Examples/MistDemo/Sources/MistDemoKit/Constants/MistDemoConstants.swift +++ b/Examples/MistDemo/Sources/MistDemoKit/Constants/MistDemoConstants.swift @@ -81,6 +81,8 @@ public enum MistDemoConstants { public static let operationsFile = "operations.file" /// Atomic configuration key. public static let atomic = "atomic" + /// Batch size configuration key for the auto-chunking `*-all` commands. + public static let batchSize = "batch.size" } // MARK: - Field Names diff --git a/Examples/MistDemo/Sources/MistDemoKit/MistDemoRunner.swift b/Examples/MistDemo/Sources/MistDemoKit/MistDemoRunner.swift index 554d1c7c..2817dc15 100644 --- a/Examples/MistDemo/Sources/MistDemoKit/MistDemoRunner.swift +++ b/Examples/MistDemo/Sources/MistDemoKit/MistDemoRunner.swift @@ -58,6 +58,8 @@ public enum MistDemoRunner { await registry.register(ListZonesCommand.self) await registry.register(ModifyZonesCommand.self) await registry.register(DiscoverCommand.self) + await registry.register(LookupAllRecordsCommand.self) + await registry.register(DiscoverAllUserIdentitiesCommand.self) await registry.register(ValidateCommand.self) await registry.register(DeleteZoneCommand.self) await registry.register(FetchChangesCommand.self) diff --git a/Sources/MistKit/CloudKitService/CloudKitService+BatchChunking.swift b/Sources/MistKit/CloudKitService/CloudKitService+BatchChunking.swift new file mode 100644 index 00000000..2733e343 --- /dev/null +++ b/Sources/MistKit/CloudKitService/CloudKitService+BatchChunking.swift @@ -0,0 +1,82 @@ +// +// CloudKitService+BatchChunking.swift +// MistKit +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +extension CloudKitService { + /// Split `items` into batches of at most `batchSize`, invoke `perBatch` + /// for each batch in order, and concatenate the results. + /// + /// This is the shared engine behind the auto-chunking convenience methods + /// (`lookupAllRecords`, `discoverAllUserIdentities(lookupInfos:)`, etc.) that + /// sit on top of CloudKit's single-request batch primitives. CloudKit caps + /// most batch endpoints at ``maxRecordsPerRequest`` items per request, so a + /// caller with a larger input must split it across multiple requests. + /// + /// Unlike server-driven pagination, the number of batches here is fully + /// determined up front (`ceil(items.count / batchSize)`), so there is no + /// runaway-loop risk and therefore no `maxPages`-style ceiling that throws — + /// `batchSize` is the only knob, clamped to `1...maxRecordsPerRequest`. + /// + /// - Parameters: + /// - items: The full input to process; an empty input issues no requests. + /// - batchSize: Maximum items per batch, clamped to + /// `1...maxRecordsPerRequest`. + /// - context: Operation label used when mapping cancellation to + /// `CloudKitError`. + /// - perBatch: Performs one batch (a single CloudKit request) and returns + /// that batch's results. + /// - Returns: Every batch's results concatenated in input order. + internal func chunkedBatches( + _ items: [Input], + batchSize: Int, + context: String, + _ perBatch: ([Input]) async throws(CloudKitError) -> [Output] + ) async throws(CloudKitError) -> [Output] { + guard !items.isEmpty else { + return [] + } + + let size = min(max(batchSize, 1), CloudKitService.maxRecordsPerRequest) + var results: [Output] = [] + var index = 0 + + while index < items.count { + do { + try Task.checkCancellation() + } catch { + throw mapToCloudKitError(error, context: context) + } + + let end = min(index + size, items.count) + results.append(contentsOf: try await perBatch(Array(items[index.. [RecordResult] { + try await chunkedBatches( + recordNames, + batchSize: batchSize, + context: "lookupAllRecords" + ) { batch throws(CloudKitError) in + try await self.lookupRecords( + recordNames: batch, + desiredKeys: desiredKeys, + database: database + ) + } + } +} diff --git a/Sources/MistKit/CloudKitService/CloudKitService+LookupOperations.swift b/Sources/MistKit/CloudKitService/CloudKitService+LookupOperations.swift index 91c14df8..c8a6d0cf 100644 --- a/Sources/MistKit/CloudKitService/CloudKitService+LookupOperations.swift +++ b/Sources/MistKit/CloudKitService/CloudKitService+LookupOperations.swift @@ -50,6 +50,10 @@ extension CloudKitService { /// /// - Note: Pass `desiredKeys` to limit which fields come back. Useful /// for list views that only need a projection. + /// - Note: This is the single-request primitive — CloudKit caps it at + /// ``maxRecordsPerRequest`` record names. For larger inputs use + /// ``lookupAllRecords(recordNames:desiredKeys:database:batchSize:)``, which + /// chunks automatically. /// - Returns: A ``RecordResult`` per requested record — `.success` for a found /// record, `.failure` (e.g. `NOT_FOUND`) for one CloudKit could not return. public func lookupRecords( diff --git a/Sources/MistKit/CloudKitService/CloudKitService+UserIdentityChunking.swift b/Sources/MistKit/CloudKitService/CloudKitService+UserIdentityChunking.swift new file mode 100644 index 00000000..2e8c6ea5 --- /dev/null +++ b/Sources/MistKit/CloudKitService/CloudKitService+UserIdentityChunking.swift @@ -0,0 +1,63 @@ +// +// CloudKitService+UserIdentityChunking.swift +// MistKit +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +extension CloudKitService { + /// Discover user identities for the given lookup infos, automatically + /// chunking large inputs. + /// + /// Convenience over ``discoverUserIdentities(lookupInfos:)`` that splits + /// `lookupInfos` into batches of at most `batchSize` (CloudKit caps a single + /// `users/discover` request at ``maxRecordsPerRequest`` infos) and + /// concatenates the per-batch identities in input order. + /// + /// - Note: This overloads the (input-less) address-book discovery + /// `discoverAllUserIdentities()` — *this* form discovers identities matching + /// the supplied `lookupInfos` via the batched POST `users/discover` + /// endpoint, whereas the no-argument form enumerates the caller's address + /// book via GET. + /// + /// - Parameters: + /// - lookupInfos: Email/phone/record-name lookups to resolve (any length). + /// - batchSize: Maximum infos per request, clamped to + /// `1...maxRecordsPerRequest` (defaults to ``maxRecordsPerRequest``). + /// - Returns: The resolved identities across all batches, in input order. + /// - Throws: `CloudKitError` if any batch fails. + public func discoverAllUserIdentities( + lookupInfos: [UserIdentityLookupInfo], + batchSize: Int = CloudKitService.maxRecordsPerRequest + ) async throws(CloudKitError) -> [UserIdentity] { + try await chunkedBatches( + lookupInfos, + batchSize: batchSize, + context: "discoverAllUserIdentities" + ) { batch throws(CloudKitError) in + try await self.discoverUserIdentities(lookupInfos: batch) + } + } +} diff --git a/Sources/MistKit/CloudKitService/CloudKitService+UserOperations.swift b/Sources/MistKit/CloudKitService/CloudKitService+UserOperations.swift index 37941edd..b958b79a 100644 --- a/Sources/MistKit/CloudKitService/CloudKitService+UserOperations.swift +++ b/Sources/MistKit/CloudKitService/CloudKitService+UserOperations.swift @@ -143,6 +143,11 @@ extension CloudKitService { /// /// Hits CloudKit's POST `users/discover` endpoint. Routed against the public /// database with web-auth credentials. + /// + /// - Note: This is the single-request primitive — CloudKit caps it at + /// ``maxRecordsPerRequest`` lookup infos. For larger inputs use + /// ``discoverAllUserIdentities(lookupInfos:batchSize:)``, which chunks + /// automatically. public func discoverUserIdentities( lookupInfos: [UserIdentityLookupInfo] ) async throws(CloudKitError) -> [UserIdentity] { diff --git a/Sources/MistKit/CloudKitService/CloudKitService.swift b/Sources/MistKit/CloudKitService/CloudKitService.swift index cdfcbddc..3deb7860 100644 --- a/Sources/MistKit/CloudKitService/CloudKitService.swift +++ b/Sources/MistKit/CloudKitService/CloudKitService.swift @@ -59,8 +59,11 @@ public struct CloudKitService: Sendable { public static let baseURL = URL(string: "https://api.apple-cloudkit.com")! // swiftlint:enable force_unwrapping - /// CloudKit's maximum number of records returned per query/modify request. - internal static let maxRecordsPerRequest: Int = 200 + /// CloudKit's maximum number of items (records, lookups, or operations) + /// accepted or returned per batch request. The auto-chunking convenience + /// methods (e.g. ``lookupAllRecords(recordNames:desiredKeys:database:batchSize:)``) + /// default their `batchSize` to this value. + public static let maxRecordsPerRequest: Int = 200 /// CloudKit's documented per-record field-data limit (1 MB). Assets travel /// via the CDN and don't count against this limit. MistKit does not diff --git a/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+Helpers.swift b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+Helpers.swift new file mode 100644 index 00000000..70237884 --- /dev/null +++ b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+Helpers.swift @@ -0,0 +1,120 @@ +// +// CloudKitServiceTests.BatchChunking+Helpers.swift +// MistKit +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +internal import Foundation +internal import HTTPTypes +internal import Testing + +@testable import MistKit + +extension CloudKitServiceTests.BatchChunking { + private static let testAPIToken = TestConstants.apiToken + + /// A `CloudKitService` backed by `provider`, configured with web-auth + /// credentials (so `.private` and `users/*` routes both work). + internal static func makeUserService( + provider: ResponseProvider + ) throws -> CloudKitService { + let transport = MockTransport(responseProvider: provider) + return try CloudKitService( + containerIdentifier: TestConstants.serviceContainerIdentifier, + credentials: Credentials( + apiAuth: APICredentials( + apiToken: testAPIToken, + webAuthToken: TestConstants.webAuthToken + ) + ), + transport: transport + ) + } + + // MARK: - Request body inspection + + /// Number of items in each recorded request body, reading the array under + /// `key` (e.g. `"records"`, `"users"`, `"lookupInfos"`). + internal static func itemCounts(in bodies: [Data?], key: String) -> [Int] { + bodies.map { data in + guard let data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let items = json[key] as? [[String: Any]] + else { + return 0 + } + return items.count + } + } + + /// Concatenate the `field`-valued strings of every item across all bodies, + /// reading items under `key` — used to verify input order is preserved. + internal static func orderedValues( + in bodies: [Data?], + key: String, + field: String + ) -> [String] { + bodies.flatMap { data -> [String] in + guard let data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let items = json[key] as? [[String: Any]] + else { + return [] + } + return items.compactMap { $0[field] as? String } + } + } +} + +// MARK: - lookupRecords Response Builder + +extension ResponseConfig { + internal static func successfulLookupRecordsResponse( + recordCount: Int = 1 + ) throws -> ResponseConfig { + var records: [[String: Any]] = [] + for index in 0.. (CloudKitService, ResponseProvider) { + let provider = ResponseProvider( + defaultResponse: try .successfulLookupRecordsResponse(recordCount: recordsPerCall) + ) + let service = try CloudKitServiceTests.BatchChunking.makeUserService( + provider: provider + ) + return (service, provider) + } + + @Test("issues one request and passes results through for a single batch") + internal func singleBatch() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let names = (0..<5).map { "rec-\($0)" } + + let results = try await service.lookupAllRecords( + recordNames: names, + database: .private + ) + + let count = await provider.callCount(for: "lookupRecords") + let sizes = CloudKitServiceTests.BatchChunking.itemCounts( + in: await provider.bodies(for: "lookupRecords"), + key: "records" + ) + #expect(count == 1) + #expect(sizes == [5]) + // The mock returns `recordsPerCall` records per request regardless of + // input size, so the result count reflects the mock, not `names.count`. + #expect(results.count == Self.recordsPerCall) + } + + @Test("accumulates results across multiple batches and preserves order") + internal func multiBatchAccumulation() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let names = (0..<450).map { "rec-\($0)" } + + let results = try await service.lookupAllRecords( + recordNames: names, + database: .private + ) + + let bodies = await provider.bodies(for: "lookupRecords") + let sizes = CloudKitServiceTests.BatchChunking.itemCounts(in: bodies, key: "records") + let order = CloudKitServiceTests.BatchChunking.orderedValues( + in: bodies, + key: "records", + field: "recordName" + ) + #expect(await provider.callCount(for: "lookupRecords") == 3) + #expect(sizes == [200, 200, 50]) + #expect(order == names) + #expect(results.count == 3 * Self.recordsPerCall) + } + + @Test("custom batchSize controls chunk boundaries") + internal func customBatchSize() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let names = (0..<5).map { "rec-\($0)" } + + _ = try await service.lookupAllRecords( + recordNames: names, + database: .private, + batchSize: 2 + ) + + let sizes = CloudKitServiceTests.BatchChunking.itemCounts( + in: await provider.bodies(for: "lookupRecords"), + key: "records" + ) + #expect(sizes == [2, 2, 1]) + } + + @Test("batchSize below 1 clamps to 1") + internal func batchSizeClampsLow() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let names = (0..<3).map { "rec-\($0)" } + + _ = try await service.lookupAllRecords( + recordNames: names, + database: .private, + batchSize: 0 + ) + + let sizes = CloudKitServiceTests.BatchChunking.itemCounts( + in: await provider.bodies(for: "lookupRecords"), + key: "records" + ) + #expect(sizes == [1, 1, 1]) + } + + @Test("batchSize above the cap clamps to maxRecordsPerRequest") + internal func batchSizeClampsHigh() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let names = (0..<250).map { "rec-\($0)" } + + _ = try await service.lookupAllRecords( + recordNames: names, + database: .private, + batchSize: 9_999 + ) + + let sizes = CloudKitServiceTests.BatchChunking.itemCounts( + in: await provider.bodies(for: "lookupRecords"), + key: "records" + ) + #expect(sizes == [200, 50]) + } + + @Test("empty input issues no requests") + internal func emptyInput() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + + let results = try await service.lookupAllRecords( + recordNames: [], + database: .private + ) + + #expect(results.isEmpty) + #expect(await provider.callCount(for: "lookupRecords") == 0) + } + + @Test("a failing batch throws and stops the loop") + internal func failingBatchPropagates() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let provider = ResponseProvider( + defaultResponse: try .successfulLookupRecordsResponse(recordCount: Self.recordsPerCall) + ) + // First batch succeeds, second fails: the loop must stop on the error + // rather than issuing the remaining batch. + await provider.enqueue( + try .successfulLookupRecordsResponse(recordCount: Self.recordsPerCall), + for: "lookupRecords" + ) + await provider.enqueue(.authenticationError(), for: "lookupRecords") + let service = try CloudKitServiceTests.BatchChunking.makeUserService( + provider: provider + ) + let names = (0..<450).map { "rec-\($0)" } + + await #expect(throws: CloudKitError.self) { + _ = try await service.lookupAllRecords( + recordNames: names, + database: .private + ) + } + // Two batches issued (success then failure); the third is never sent. + #expect(await provider.callCount(for: "lookupRecords") == 2) + } + } +} diff --git a/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+UserIdentityChunking.swift b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+UserIdentityChunking.swift new file mode 100644 index 00000000..43e9b409 --- /dev/null +++ b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking+UserIdentityChunking.swift @@ -0,0 +1,108 @@ +// +// CloudKitServiceTests.BatchChunking+UserIdentityChunking.swift +// MistKit +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +internal import Foundation +internal import Testing + +@testable import MistKit + +extension CloudKitServiceTests.BatchChunking { + @Suite("User-identity chunking") + internal struct UserIdentityChunking { + private static let identitiesPerCall = 2 + + private static func makeService() throws -> (CloudKitService, ResponseProvider) { + let provider = ResponseProvider( + defaultResponse: try .successfulDiscoverUserIdentitiesResponse( + identityCount: identitiesPerCall + ) + ) + let service = try CloudKitServiceTests.BatchChunking.makeUserService(provider: provider) + return (service, provider) + } + + // MARK: - discoverAllUserIdentities(lookupInfos:) + + @Test("discoverAllUserIdentities chunks, accumulates, and preserves order") + internal func discoverMultiBatch() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let infos = (0..<450).map { UserIdentityLookupInfo(userRecordName: "_user-\($0)") } + + let identities = try await service.discoverAllUserIdentities(lookupInfos: infos) + + let bodies = await provider.bodies(for: "discoverUserIdentities") + let sizes = CloudKitServiceTests.BatchChunking.itemCounts(in: bodies, key: "lookupInfos") + let order = CloudKitServiceTests.BatchChunking.orderedValues( + in: bodies, + key: "lookupInfos", + field: "userRecordName" + ) + #expect(await provider.callCount(for: "discoverUserIdentities") == 3) + #expect(sizes == [200, 200, 50]) + #expect(order == infos.map { $0.userRecordName ?? "" }) + #expect(identities.count == 3 * Self.identitiesPerCall) + } + + @Test("discoverAllUserIdentities clamps batchSize to the per-request cap") + internal func discoverClampsBatchSize() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + let infos = (0..<250).map { UserIdentityLookupInfo(userRecordName: "_user-\($0)") } + + _ = try await service.discoverAllUserIdentities(lookupInfos: infos, batchSize: 9_999) + + let sizes = CloudKitServiceTests.BatchChunking.itemCounts( + in: await provider.bodies(for: "discoverUserIdentities"), + key: "lookupInfos" + ) + #expect(sizes == [200, 50]) + } + + @Test("discoverAllUserIdentities issues no requests for empty input") + internal func discoverEmptyInput() async throws { + guard #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) else { + Issue.record("CloudKitService is not available on this operating system.") + return + } + let (service, provider) = try Self.makeService() + + let identities = try await service.discoverAllUserIdentities(lookupInfos: []) + + #expect(identities.isEmpty) + #expect(await provider.callCount(for: "discoverUserIdentities") == 0) + } + } +} diff --git a/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking.swift b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking.swift new file mode 100644 index 00000000..785eb162 --- /dev/null +++ b/Tests/MistKitTests/CloudKitService/BatchChunking/CloudKitServiceTests.BatchChunking.swift @@ -0,0 +1,38 @@ +// +// CloudKitServiceTests.BatchChunking.swift +// MistKit +// +// Created by Leo Dion. +// Copyright © 2026 BrightDigit. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +internal import Foundation +internal import Testing + +@testable import MistKit + +extension CloudKitServiceTests { + @Suite("CloudKitService Batch Chunking", .enabled(if: Platform.isCryptoAvailable)) + internal enum BatchChunking {} +} diff --git a/Tests/MistKitTests/Mocks/MockTransport.swift b/Tests/MistKitTests/Mocks/MockTransport.swift index e6e03d4d..34cf0647 100644 --- a/Tests/MistKitTests/Mocks/MockTransport.swift +++ b/Tests/MistKitTests/Mocks/MockTransport.swift @@ -45,6 +45,16 @@ internal struct MockTransport: ClientTransport, Sendable { baseURL: URL, operationID: String ) async throws -> (HTTPResponse, HTTPBody?) { - try await responseProvider.response(for: operationID, request: request) + let bodyData: Data? = + if let body { + try await Data(collecting: body, upTo: 10 * 1_024 * 1_024) + } else { + nil + } + return try await responseProvider.response( + for: operationID, + request: request, + body: bodyData + ) } } diff --git a/Tests/MistKitTests/Mocks/ResponseProvider.swift b/Tests/MistKitTests/Mocks/ResponseProvider.swift index 7d7bf6d5..038cd44a 100644 --- a/Tests/MistKitTests/Mocks/ResponseProvider.swift +++ b/Tests/MistKitTests/Mocks/ResponseProvider.swift @@ -27,6 +27,7 @@ // OTHER DEALINGS IN THE SOFTWARE. // +internal import Foundation internal import HTTPTypes internal import OpenAPIRuntime @@ -45,6 +46,9 @@ internal actor ResponseProvider { private var defaultResponse: ResponseConfig private var responseQueues: [String: [ResponseConfig]] = [:] + /// Every request received, in order, captured for test assertions. + private var requestLog: [(operationID: String, body: Data?)] = [] + // MARK: - Initializers internal init( @@ -83,10 +87,25 @@ internal actor ResponseProvider { responseQueues[operationID, default: []].append(response) } + // MARK: - Request Inspection + + /// Number of requests received for `operationID`. + internal func callCount(for operationID: String) -> Int { + requestLog.filter { $0.operationID == operationID }.count + } + + /// The request bodies received for `operationID`, in order. + internal func bodies(for operationID: String) -> [Data?] { + requestLog.filter { $0.operationID == operationID }.map(\.body) + } + internal func response( for operationID: String, - request _: HTTPRequest + request _: HTTPRequest, + body: Data? = nil ) throws -> (HTTPResponse, HTTPBody?) { + requestLog.append((operationID: operationID, body: body)) + let config: ResponseConfig if var queue = responseQueues[operationID], !queue.isEmpty { config = queue.removeFirst()