Adapters: add Meet, Zoom, Teams (coloured border) + adapter registry
Front-loads the remaining visual adapters per the Signal→Meet→Zoom priority. All three reuse GridCallAnalyzer's coloured-border (saturated) detection path and share the new bottom-left name anchor: - GridCallAnalyzer: generalise nameAtBottom:Bool into a NameAnchor enum (.bottomCenter for Signal's centered footer, .bottomLeft for Meet/Zoom/Teams where the name hugs the tile's bottom-left corner, .center for completeness). tileRect estimates the tile up-and-right of a bottom-left name. - MeetAdapter (Google-blue ring, browser-hosted), ZoomAdapter (green/yellow border), TeamsAdapter (violet ring): coloured-border on, white-border off, bottom-left names. Geometry constants are first-pass pending real fixtures. - AdapterRegistry.adapter(for:) maps CallDetector.DetectedApp → AppAdapter so VisualObserver can be constructed when live visual capture is wired in; unmapped apps degrade to audio-only. Synthetic 4-tile tests: Meet picks each blue-bordered speaker with no adjacent-tile bleed, Zoom picks the green-bordered speaker, and Signal's white-only detector correctly ignores a coloured border. 18/18 XCTest pass.
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
import Foundation
|
||||
|
||||
/// Maps a detected call app to its screen-reading adapter. One place to wire new
|
||||
/// adapters; `VisualObserver` is constructed from whatever this returns when the
|
||||
/// live visual capture is integrated into the session.
|
||||
enum AdapterRegistry {
|
||||
/// The adapter for a detected app, or nil if that app has no visual adapter
|
||||
/// yet (the session then runs audio-only — graceful degradation).
|
||||
static func adapter(for app: CallDetector.DetectedApp) -> (any AppAdapter)? {
|
||||
switch app {
|
||||
case .signal: return SignalAdapter()
|
||||
case .meet: return MeetAdapter()
|
||||
case .zoom: return ZoomAdapter()
|
||||
case .teams: return TeamsAdapter()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import Foundation
|
||||
import CoreVideo
|
||||
|
||||
/// Google Meet adapter (browser tab — capture is at the browser-window level).
|
||||
///
|
||||
/// Meet's active-speaker cue is a **coloured (Google-blue) ring/glow** around the
|
||||
/// speaking participant's tile, plus animated audio bars in the tile's mic chip.
|
||||
/// The participant **name sits in the tile's bottom-LEFT corner**, so the tile is
|
||||
/// estimated extending up and to the right of the name.
|
||||
///
|
||||
/// Detection *logic* is validated on synthetic frames; the geometry constants are a
|
||||
/// first pass and will be calibrated against real Meet screenshots. Meet runs in a
|
||||
/// browser, so there's no Accessibility name source we rely on — OCR only.
|
||||
struct MeetAdapter: AppAdapter {
|
||||
// Browsers that can host a Meet tab. The window, not the app, is what we capture;
|
||||
// CallDetector decides a browser window is a Meet call by its title.
|
||||
static let bundleIDs = [
|
||||
"com.google.Chrome", "org.mozilla.firefox", "com.apple.Safari",
|
||||
"company.thebrowser.Browser", "com.brave.Browser", "com.microsoft.edgemac",
|
||||
"com.google.Chrome.canary", "org.chromium.Chromium",
|
||||
]
|
||||
let adapterVersion = "meet-0.1.0"
|
||||
let preferredFPS = 3
|
||||
|
||||
private let analyzer: GridCallAnalyzer
|
||||
|
||||
init() {
|
||||
var config = GridCallAnalyzer.Config()
|
||||
config.nameAnchor = .bottomLeft
|
||||
config.detectColoredBorder = true // Google-blue speaking ring
|
||||
config.detectWhiteBorder = false
|
||||
config.tileExpandX = 3.0
|
||||
config.tileExpandY = 5.0
|
||||
self.analyzer = GridCallAnalyzer(config: config)
|
||||
}
|
||||
|
||||
func analyze(frame: CVPixelBuffer, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(pixelBuffer: frame, at: t)
|
||||
}
|
||||
|
||||
// Exposed for fixture/synthetic tests.
|
||||
func analyze(cgImage: CGImage, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(cgImage: cgImage, at: t)
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ struct SignalAdapter: AppAdapter {
|
||||
// with real fixtures. (Gotchas, per Signal source: NO border in 1:1 calls —
|
||||
// fall back to mic-VAD/audio pill — and in Speaker view the large tile is
|
||||
// the speaker; both handled at a higher level later.)
|
||||
config.nameAtBottom = true
|
||||
config.nameAnchor = .bottomCenter
|
||||
config.detectWhiteBorder = true
|
||||
config.detectColoredBorder = false
|
||||
config.tileExpandX = 2.4
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
import CoreVideo
|
||||
|
||||
/// Microsoft Teams adapter (native app: com.microsoft.teams2 / .teams).
|
||||
///
|
||||
/// Teams' active-speaker cue is a **coloured ring/border** (Teams violet) around
|
||||
/// the speaking participant's tile; the **name sits in the tile's bottom-left**.
|
||||
/// Not a launch priority — included so the four detected platforms all have a
|
||||
/// visual adapter and degrade no worse than colored-border detection.
|
||||
///
|
||||
/// Detection *logic* is validated on synthetic frames; geometry constants are a
|
||||
/// first pass pending real Teams screenshots.
|
||||
struct TeamsAdapter: AppAdapter {
|
||||
static let bundleIDs = ["com.microsoft.teams2", "com.microsoft.teams"]
|
||||
let adapterVersion = "teams-0.1.0"
|
||||
let preferredFPS = 3
|
||||
|
||||
private let analyzer: GridCallAnalyzer
|
||||
|
||||
init() {
|
||||
var config = GridCallAnalyzer.Config()
|
||||
config.nameAnchor = .bottomLeft
|
||||
config.detectColoredBorder = true // Teams-violet speaking ring
|
||||
config.detectWhiteBorder = false
|
||||
config.tileExpandX = 3.0
|
||||
config.tileExpandY = 5.0
|
||||
self.analyzer = GridCallAnalyzer(config: config)
|
||||
}
|
||||
|
||||
func analyze(frame: CVPixelBuffer, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(pixelBuffer: frame, at: t)
|
||||
}
|
||||
|
||||
// Exposed for fixture/synthetic tests.
|
||||
func analyze(cgImage: CGImage, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(cgImage: cgImage, at: t)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import Foundation
|
||||
import CoreVideo
|
||||
|
||||
/// Zoom adapter (native app: us.zoom.xos).
|
||||
///
|
||||
/// Zoom's active-speaker cue is a **coloured border** around the speaking tile —
|
||||
/// by default a green/yellow outline (configurable in Zoom settings). The
|
||||
/// participant **name sits in the tile's bottom-LEFT corner**, so the tile is
|
||||
/// estimated extending up and to the right of the name.
|
||||
///
|
||||
/// Gotchas to calibrate against real fixtures later:
|
||||
/// - **Speaker view** shows one big tile; the active speaker fills it (no useful
|
||||
/// per-tile border) — handle by attributing speech to the large tile.
|
||||
/// - The self-view tile and screen-share change the layout.
|
||||
///
|
||||
/// Detection *logic* is validated on synthetic frames; geometry constants are a
|
||||
/// first pass pending real Zoom screenshots.
|
||||
struct ZoomAdapter: AppAdapter {
|
||||
static let bundleIDs = ["us.zoom.xos"]
|
||||
let adapterVersion = "zoom-0.1.0"
|
||||
let preferredFPS = 3
|
||||
|
||||
private let analyzer: GridCallAnalyzer
|
||||
|
||||
init() {
|
||||
var config = GridCallAnalyzer.Config()
|
||||
config.nameAnchor = .bottomLeft
|
||||
config.detectColoredBorder = true // green/yellow speaking border
|
||||
config.detectWhiteBorder = false
|
||||
config.tileExpandX = 3.0
|
||||
config.tileExpandY = 5.0
|
||||
self.analyzer = GridCallAnalyzer(config: config)
|
||||
}
|
||||
|
||||
func analyze(frame: CVPixelBuffer, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(pixelBuffer: frame, at: t)
|
||||
}
|
||||
|
||||
// Exposed for fixture/synthetic tests.
|
||||
func analyze(cgImage: CGImage, at t: TimeInterval) -> [SpeakerObservation] {
|
||||
analyzer.analyze(cgImage: cgImage, at: t)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user