Avi Drissman | 8ba1bad | 2022-09-13 19:22:36 | [diff] [blame] | 1 | // Copyright 2022 The Chromium Authors |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 5 | #include "services/screen_ai/screen_ai_service_impl.h" |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 6 | |
Ramin Halavati | 2667959 | 2025-03-13 07:24:57 | [diff] [blame] | 7 | #include <algorithm> |
Ramin Halavati | 55fdf4d | 2022-11-07 06:35:18 | [diff] [blame] | 8 | #include <memory> |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 9 | #include <optional> |
Ramin Halavati | fd712d8e | 2022-10-19 05:11:27 | [diff] [blame] | 10 | #include <utility> |
Ramin Halavati | b044504d | 2022-10-24 06:36:34 | [diff] [blame] | 11 | #include <vector> |
Ramin Halavati | fd712d8e | 2022-10-19 05:11:27 | [diff] [blame] | 12 | |
| 13 | #include "base/check.h" |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 14 | #include "base/check_is_test.h" |
Tom Sepez | e9b4dd55 | 2024-08-14 22:44:29 | [diff] [blame] | 15 | #include "base/compiler_specific.h" |
Ramin Halavati | 65f96fb | 2025-02-24 16:45:18 | [diff] [blame] | 16 | #include "base/cpu.h" |
Ramin Halavati | 55fdf4d | 2022-11-07 06:35:18 | [diff] [blame] | 17 | #include "base/functional/bind.h" |
| 18 | #include "base/location.h" |
Ramin Halavati | fafff15 | 2022-12-05 09:39:38 | [diff] [blame] | 19 | #include "base/logging.h" |
Kyungjun Lee | 9d755a0 | 2022-11-08 17:57:11 | [diff] [blame] | 20 | #include "base/metrics/histogram_functions.h" |
Ramin Halavati | 5ff855c | 2022-04-27 16:30:48 | [diff] [blame] | 21 | #include "base/process/process.h" |
Ramin Halavati | 65f96fb | 2025-02-24 16:45:18 | [diff] [blame] | 22 | #include "base/strings/stringprintf.h" |
Ramin Halavati | 2667959 | 2025-03-13 07:24:57 | [diff] [blame] | 23 | #include "base/system/sys_info.h" |
Sean Maher | 5b9af51f | 2022-11-21 15:32:47 | [diff] [blame] | 24 | #include "base/task/single_thread_task_runner.h" |
Ramin Halavati | 55fdf4d | 2022-11-07 06:35:18 | [diff] [blame] | 25 | #include "base/task/thread_pool.h" |
Ramin Halavati | 65f96fb | 2025-02-24 16:45:18 | [diff] [blame] | 26 | #include "components/crash/core/common/crash_key.h" |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 27 | #include "services/screen_ai/buildflags/buildflags.h" |
Ramin Halavati | 4c47eb8 | 2024-12-03 19:06:42 | [diff] [blame] | 28 | #include "services/screen_ai/proto/chrome_screen_ai.pb.h" |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 29 | #include "services/screen_ai/proto/main_content_extractor_proto_convertor.h" |
| 30 | #include "services/screen_ai/proto/visual_annotator_proto_convertor.h" |
Ramin Halavati | 4c47eb8 | 2024-12-03 19:06:42 | [diff] [blame] | 31 | #include "services/screen_ai/public/cpp/metrics.h" |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 32 | #include "services/screen_ai/public/cpp/utilities.h" |
Ramin Halavati | eddadb6 | 2022-05-04 17:29:49 | [diff] [blame] | 33 | #include "ui/accessibility/accessibility_features.h" |
Ramin Halavati | 4f904d0a | 2024-06-25 15:32:32 | [diff] [blame] | 34 | #include "ui/accessibility/ax_node.h" |
| 35 | #include "ui/accessibility/ax_tree.h" |
Ramin Halavati | 8423107 | 2022-08-17 08:02:20 | [diff] [blame] | 36 | #include "ui/accessibility/ax_tree_id.h" |
Nektarios Paisios | f73d697 | 2022-06-04 11:32:25 | [diff] [blame] | 37 | #include "ui/gfx/geometry/rect_f.h" |
Ramin Halavati | 367352a | 2022-04-14 06:00:23 | [diff] [blame] | 38 | |
Ramin Halavati | a9b50a10 | 2024-02-07 18:37:23 | [diff] [blame] | 39 | #if BUILDFLAG(USE_FAKE_SCREEN_AI) |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 40 | #include "services/screen_ai/screen_ai_library_wrapper_fake.h" |
Ramin Halavati | 767b8fc | 2024-02-02 06:23:08 | [diff] [blame] | 41 | #else |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 42 | #include "services/screen_ai/screen_ai_library_wrapper_impl.h" |
Ramin Halavati | 767b8fc | 2024-02-02 06:23:08 | [diff] [blame] | 43 | #endif |
| 44 | |
Ramin Halavati | 69bf775 | 2022-04-04 09:58:15 | [diff] [blame] | 45 | namespace screen_ai { |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 46 | |
Ramin Halavati | 6d4fc2b | 2022-06-16 15:33:35 | [diff] [blame] | 47 | namespace { |
| 48 | |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 49 | // Maximum image resolution that OCR service processes. Images larger than this |
| 50 | // threshold are downsampled before processing. |
| 51 | const uint32_t kLargestOcrResolution = 2048 * 2048; |
| 52 | |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 53 | // How often it would be checked that the service is idle and can be shutdown. |
Ramin Halavati | dc0b2e51 | 2024-08-16 17:39:09 | [diff] [blame] | 54 | constexpr base::TimeDelta kIdleCheckingDelay = base::Minutes(5); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 55 | |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 56 | // How long after all clients are disconnected, it is checked if service is |
| 57 | // idle. |
Ramin Halavati | 2fa45b8 | 2024-08-15 16:46:53 | [diff] [blame] | 58 | constexpr base::TimeDelta kCoolDownTime = base::Seconds(10); |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 59 | |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 60 | // These values are persisted to logs. Entries should not be renumbered and |
| 61 | // numeric values should never be reused. |
Ramin Halavati | 29cc115 | 2024-10-27 17:30:28 | [diff] [blame] | 62 | // See `screen_ai_service.mojom` for more info. |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 63 | // LINT.IfChange(OcrClientType) |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 64 | enum class OcrClientTypeForMetrics { |
| 65 | kTest = 0, |
| 66 | kPdfViewer = 1, |
| 67 | kLocalSearch = 2, |
| 68 | kCameraApp = 3, |
Ramin Halavati | 29cc115 | 2024-10-27 17:30:28 | [diff] [blame] | 69 | kNotUsed = 4, // Can be used for a new client. |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 70 | kMediaApp = 5, |
Michelle Chen | 5192d52 | 2024-10-22 23:38:22 | [diff] [blame] | 71 | kScreenshotTextDetection, |
| 72 | kMaxValue = kScreenshotTextDetection |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 73 | }; |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 74 | // LINT.ThenChange(//tools/metrics/histograms/metadata/accessibility/enums.xml:OcrClientType) |
| 75 | |
| 76 | // These values are persisted to logs. Entries should not be renumbered and |
| 77 | // numeric values should never be reused. |
| 78 | // See `screen_ai_service.mojom` for more info. |
| 79 | // LINT.IfChange(MainContentExtractionClientType) |
| 80 | enum class MainContentExtractionClientTypeForMetrics { |
| 81 | kTest = 0, |
| 82 | kReadingMode = 1, |
| 83 | kMainNode = 2, |
| 84 | kMahi = 3, |
| 85 | kMaxValue = kMahi |
| 86 | }; |
| 87 | // LINT.ThenChange(//tools/metrics/histograms/metadata/accessibility/enums.xml:MainContentExtractionClientType) |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 88 | |
| 89 | OcrClientTypeForMetrics GetClientType(mojom::OcrClientType client_type) { |
| 90 | switch (client_type) { |
| 91 | case mojom::OcrClientType::kTest: |
| 92 | CHECK_IS_TEST(); |
| 93 | return OcrClientTypeForMetrics::kTest; |
| 94 | case mojom::OcrClientType::kPdfViewer: |
| 95 | return OcrClientTypeForMetrics::kPdfViewer; |
| 96 | case mojom::OcrClientType::kLocalSearch: |
| 97 | return OcrClientTypeForMetrics::kLocalSearch; |
| 98 | case mojom::OcrClientType::kCameraApp: |
| 99 | return OcrClientTypeForMetrics::kCameraApp; |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 100 | case mojom::OcrClientType::kMediaApp: |
| 101 | return OcrClientTypeForMetrics::kMediaApp; |
Michelle Chen | 5192d52 | 2024-10-22 23:38:22 | [diff] [blame] | 102 | case mojom::OcrClientType::kScreenshotTextDetection: |
| 103 | return OcrClientTypeForMetrics::kScreenshotTextDetection; |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 104 | } |
| 105 | } |
| 106 | |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 107 | MainContentExtractionClientTypeForMetrics GetClientType( |
| 108 | mojom::MceClientType client_type) { |
| 109 | switch (client_type) { |
| 110 | case mojom::MceClientType::kTest: |
| 111 | CHECK_IS_TEST(); |
| 112 | return MainContentExtractionClientTypeForMetrics::kTest; |
| 113 | case mojom::MceClientType::kReadingMode: |
| 114 | return MainContentExtractionClientTypeForMetrics::kReadingMode; |
| 115 | case mojom::MceClientType::kMainNode: |
| 116 | return MainContentExtractionClientTypeForMetrics::kMainNode; |
| 117 | case mojom::MceClientType::kMahi: |
| 118 | return MainContentExtractionClientTypeForMetrics::kMahi; |
| 119 | } |
| 120 | } |
| 121 | |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 122 | ui::AXTreeUpdate ConvertVisualAnnotationToTreeUpdate( |
Ramin Halavati | dcb09a17 | 2024-07-15 08:28:55 | [diff] [blame] | 123 | std::optional<chrome_screen_ai::VisualAnnotation>& annotation_proto, |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 124 | const gfx::Rect& image_rect) { |
| 125 | if (!annotation_proto) { |
| 126 | VLOG(0) << "Screen AI library could not process snapshot or no OCR data."; |
| 127 | return ui::AXTreeUpdate(); |
| 128 | } |
| 129 | |
| 130 | return VisualAnnotationToAXTreeUpdate(*annotation_proto, image_rect); |
| 131 | } |
| 132 | |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 133 | ui::AXNodeID ComputeMainNode( |
| 134 | const ui::AXTree* tree, |
| 135 | const std::vector<ui::AXNodeID>& content_node_ids) { |
| 136 | ui::AXNode* front = tree->GetFromId(content_node_ids.front()); |
| 137 | ui::AXNode* back = tree->GetFromId(content_node_ids.back()); |
| 138 | ui::AXNode* main = front->GetLowestCommonAncestor(*back); |
| 139 | return main->id(); |
| 140 | } |
| 141 | |
Ramin Halavati | 65f96fb | 2025-02-24 16:45:18 | [diff] [blame] | 142 | #if !BUILDFLAG(USE_FAKE_SCREEN_AI) |
| 143 | void SetCPUInstructionSetCrashKey() { |
| 144 | #if defined(ARCH_CPU_X86_FAMILY) |
| 145 | base::CPU(); |
| 146 | // Report cpu micro architecture in case of crash. |
| 147 | static crash_reporter::CrashKeyString<3> cpu_info("intel_micro_architecture"); |
| 148 | cpu_info.Set( |
| 149 | base::StringPrintf("%i", base::CPU().GetIntelMicroArchitecture())); |
| 150 | #endif |
| 151 | } |
| 152 | #endif |
| 153 | |
Ramin Halavati | 2667959 | 2025-03-13 07:24:57 | [diff] [blame] | 154 | // Return a maximum 11 character string with the signature of available and |
| 155 | // total memory, both in MB and capped to 99999. |
| 156 | std::string GetMemoryStatusForCrashKey() { |
| 157 | int total_memory = base::SysInfo::AmountOfPhysicalMemoryMB(); |
| 158 | int available_memory = static_cast<int>( |
| 159 | base::SysInfo::AmountOfAvailablePhysicalMemory() / (1024 * 1024)); |
| 160 | |
| 161 | // Cap the number of digits for crash report. |
| 162 | total_memory = std::min(total_memory, 99999); |
| 163 | available_memory = std::min(available_memory, 99999); |
| 164 | return base::StringPrintf("%i,%i", available_memory, total_memory); |
| 165 | } |
| 166 | |
Ramin Halavati | 26ae6b7 | 2022-11-10 06:45:05 | [diff] [blame] | 167 | } // namespace |
| 168 | |
Ramin Halavati | 39bab0d | 2024-01-30 06:05:01 | [diff] [blame] | 169 | // The library accepts simple pointers to model data retrieval functions, hence |
| 170 | // callback functions with linked object are not safe to pass. |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 171 | // This global variable keeps the pointer the only instance of this class. |
| 172 | ModelDataHolder* g_model_data_holder_instance = nullptr; |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 173 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 174 | // Keeps the handles of model files, and replies to calls for copying their |
| 175 | // content. |
| 176 | class ModelDataHolder { |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 177 | public: |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 178 | ModelDataHolder() { |
| 179 | CHECK(!g_model_data_holder_instance); |
| 180 | g_model_data_holder_instance = this; |
| 181 | } |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 182 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 183 | ModelDataHolder(const ModelDataHolder&) = delete; |
| 184 | ModelDataHolder& operator=(const ModelDataHolder&) = delete; |
| 185 | |
| 186 | ~ModelDataHolder() { |
| 187 | CHECK_EQ(g_model_data_holder_instance, this); |
| 188 | g_model_data_holder_instance = nullptr; |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 189 | } |
| 190 | |
| 191 | // Returns 0 if file is not found. |
| 192 | static uint32_t GetDataSize(const char* relative_file_path) { |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 193 | CHECK(g_model_data_holder_instance); |
| 194 | base::File* model_file = |
| 195 | g_model_data_holder_instance->GetModelFile(relative_file_path); |
| 196 | return model_file ? model_file->GetLength() : 0; |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 197 | } |
| 198 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 199 | // Copies content of the file in `relative_file_path` to `buffer`. Expects |
| 200 | // that `buffer_size` would be enough for the entire file content. |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 201 | static void CopyData(const char* relative_file_path, |
| 202 | uint32_t buffer_size, |
| 203 | char* buffer) { |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 204 | CHECK(g_model_data_holder_instance); |
| 205 | base::File* model_file = |
| 206 | g_model_data_holder_instance->GetModelFile(relative_file_path); |
| 207 | CHECK(model_file); |
| 208 | |
| 209 | int64_t length = model_file->GetLength(); |
| 210 | CHECK_GE(buffer_size, length); |
Tom Sepez | e9b4dd55 | 2024-08-14 22:44:29 | [diff] [blame] | 211 | CHECK_EQ(UNSAFE_TODO(model_file->Read(0, buffer, length)), length); |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 212 | } |
| 213 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 214 | void AddModelFiles(base::flat_map<base::FilePath, base::File> model_files) { |
| 215 | for (auto& model_file : model_files) { |
| 216 | model_files_[model_file.first.MaybeAsASCII()] = |
| 217 | std::move(model_file.second); |
Ramin Halavati | 39bab0d | 2024-01-30 06:05:01 | [diff] [blame] | 218 | } |
| 219 | } |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 220 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 221 | // Returns the file handle for `relative_file_path` if it exists. |
| 222 | base::File* GetModelFile(const char* relative_file_path) { |
| 223 | if (!base::Contains(model_files_, relative_file_path)) { |
| 224 | return nullptr; |
| 225 | } |
| 226 | return &model_files_[relative_file_path]; |
| 227 | } |
| 228 | |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 229 | private: |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 230 | std::map<std::string, base::File> model_files_; |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 231 | }; |
| 232 | |
Ramin Halavati | 26ae6b7 | 2022-11-10 06:45:05 | [diff] [blame] | 233 | ScreenAIService::ScreenAIService( |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 234 | mojo::PendingReceiver<mojom::ScreenAIServiceFactory> receiver) |
| 235 | : factory_receiver_(this, std::move(receiver)), |
| 236 | ocr_receiver_(this), |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 237 | main_content_extraction_receiver_(this) { |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 238 | screen2x_main_content_extractors_.set_disconnect_handler( |
| 239 | base::BindRepeating(&ScreenAIService::CheckIdleStateAfterDelay, |
| 240 | weak_ptr_factory_.GetWeakPtr())); |
| 241 | screen_ai_annotators_.set_disconnect_handler( |
| 242 | base::BindRepeating(&ScreenAIService::OcrReceiverDisconnected, |
| 243 | weak_ptr_factory_.GetWeakPtr())); |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 244 | model_data_holder_ = std::make_unique<ModelDataHolder>(); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 245 | idle_checking_timer_ = std::make_unique<base::RepeatingTimer>(); |
| 246 | idle_checking_timer_->Start(FROM_HERE, kIdleCheckingDelay, this, |
| 247 | &ScreenAIService::ShutDownIfNoClients); |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 248 | } |
Ramin Halavati | 26ae6b7 | 2022-11-10 06:45:05 | [diff] [blame] | 249 | |
| 250 | ScreenAIService::~ScreenAIService() = default; |
| 251 | |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 252 | void ScreenAIService::LoadLibrary(const base::FilePath& library_path) { |
Abigail Klein | 7a63c57 | 2024-02-28 20:45:09 | [diff] [blame] | 253 | // The ScopedBlockingCall in LoadLibrary guarantees that this is not run on |
| 254 | // the UI thread. |
Ramin Halavati | a9b50a10 | 2024-02-07 18:37:23 | [diff] [blame] | 255 | #if BUILDFLAG(USE_FAKE_SCREEN_AI) |
Ramin Halavati | 767b8fc | 2024-02-02 06:23:08 | [diff] [blame] | 256 | library_ = std::make_unique<ScreenAILibraryWrapperFake>(); |
| 257 | #else |
| 258 | library_ = std::make_unique<ScreenAILibraryWrapperImpl>(); |
Ramin Halavati | 65f96fb | 2025-02-24 16:45:18 | [diff] [blame] | 259 | |
| 260 | // TODO(crbug.com/381256355): Remove when the library is SSE3 compatible. |
| 261 | SetCPUInstructionSetCrashKey(); |
Ramin Halavati | 767b8fc | 2024-02-02 06:23:08 | [diff] [blame] | 262 | #endif |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 263 | |
| 264 | bool load_sucessful = library_->Load(library_path); |
| 265 | base::UmaHistogramBoolean("Accessibility.ScreenAI.Library.Initialized", |
| 266 | load_sucessful); |
| 267 | |
| 268 | if (!load_sucessful) { |
| 269 | library_.reset(); |
| 270 | return; |
| 271 | } |
| 272 | |
| 273 | uint32_t version_major; |
| 274 | uint32_t version_minor; |
| 275 | library_->GetLibraryVersion(version_major, version_minor); |
| 276 | VLOG(2) << "Screen AI library version: " << version_major << "." |
| 277 | << version_minor; |
| 278 | |
Georg Neis | 06e38777 | 2024-12-25 07:32:22 | [diff] [blame] | 279 | #if BUILDFLAG(IS_CHROMEOS) |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 280 | library_->SetLogger(); |
| 281 | #endif |
| 282 | |
| 283 | if (features::IsScreenAIDebugModeEnabled()) { |
| 284 | library_->EnableDebugMode(); |
| 285 | } |
| 286 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 287 | library_->SetFileContentFunctions(&ModelDataHolder::GetDataSize, |
| 288 | &ModelDataHolder::CopyData); |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 289 | } |
| 290 | |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 291 | void ScreenAIService::InitializeMainContentExtraction( |
Ramin Halavati | d139a0f | 2023-02-28 13:25:28 | [diff] [blame] | 292 | const base::FilePath& library_path, |
Ramin Halavati | fbc6202 | 2023-11-10 11:05:07 | [diff] [blame] | 293 | base::flat_map<base::FilePath, base::File> model_files, |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 294 | mojo::PendingReceiver<mojom::MainContentExtractionService> |
| 295 | main_content_extractor_service_receiver, |
| 296 | InitializeMainContentExtractionCallback callback) { |
| 297 | if (!library_) { |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 298 | LoadLibrary(library_path); |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 299 | } |
| 300 | |
| 301 | if (!library_) { |
| 302 | std::move(callback).Run(false); |
| 303 | base::Process::TerminateCurrentProcessImmediately(-1); |
| 304 | } |
| 305 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 306 | model_data_holder_->AddModelFiles(std::move(model_files)); |
Ramin Halavati | b044504d | 2022-10-24 06:36:34 | [diff] [blame] | 307 | |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 308 | bool init_successful = library_->InitMainContentExtraction(); |
Ramin Halavati | 1d57c2d | 2023-05-24 05:18:24 | [diff] [blame] | 309 | base::UmaHistogramBoolean( |
| 310 | "Accessibility.ScreenAI.MainContentExtraction.Initialized", |
| 311 | init_successful); |
| 312 | if (!init_successful) { |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 313 | std::move(callback).Run(false); |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 314 | return; |
Ramin Halavati | d139a0f | 2023-02-28 13:25:28 | [diff] [blame] | 315 | } |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 316 | |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 317 | // This interface should be created only once. |
| 318 | CHECK(!main_content_extraction_receiver_.is_bound()); |
| 319 | |
| 320 | main_content_extraction_receiver_.Bind( |
| 321 | std::move(main_content_extractor_service_receiver)); |
| 322 | |
| 323 | std::move(callback).Run(true); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 324 | main_content_extraction_last_used_ = base::TimeTicks::Now(); |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 325 | } |
| 326 | |
| 327 | void ScreenAIService::InitializeOCR( |
| 328 | const base::FilePath& library_path, |
Ramin Halavati | fbc6202 | 2023-11-10 11:05:07 | [diff] [blame] | 329 | base::flat_map<base::FilePath, base::File> model_files, |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 330 | mojo::PendingReceiver<mojom::OCRService> ocr_service_receiver, |
| 331 | InitializeOCRCallback callback) { |
Ramin Halavati | 2667959 | 2025-03-13 07:24:57 | [diff] [blame] | 332 | static crash_reporter::CrashKeyString<12> memory_ocr_init( |
| 333 | "screen_ai_mem_ocr_init"); |
| 334 | memory_ocr_init.Set(GetMemoryStatusForCrashKey()); |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 335 | if (!library_) { |
Ramin Halavati | 8ced639 | 2023-09-25 07:27:50 | [diff] [blame] | 336 | LoadLibrary(library_path); |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 337 | } |
| 338 | |
| 339 | if (!library_) { |
| 340 | std::move(callback).Run(false); |
| 341 | base::Process::TerminateCurrentProcessImmediately(-1); |
| 342 | } |
| 343 | |
Ramin Halavati | 63d9c25 | 2024-07-29 06:59:17 | [diff] [blame] | 344 | model_data_holder_->AddModelFiles(std::move(model_files)); |
| 345 | |
Ramin Halavati | 0e2ae574 | 2023-11-10 05:57:54 | [diff] [blame] | 346 | bool init_successful = library_->InitOCR(); |
Ramin Halavati | 35d129f | 2023-06-22 16:37:09 | [diff] [blame] | 347 | base::UmaHistogramBoolean("Accessibility.ScreenAI.OCR.Initialized", |
Ramin Halavati | 1d57c2d | 2023-05-24 05:18:24 | [diff] [blame] | 348 | init_successful); |
Ramin Halavati | 48058433 | 2023-08-02 06:38:40 | [diff] [blame] | 349 | |
Ramin Halavati | 1d57c2d | 2023-05-24 05:18:24 | [diff] [blame] | 350 | if (!init_successful) { |
Ramin Halavati | c1e4fa9 | 2023-05-17 17:22:06 | [diff] [blame] | 351 | std::move(callback).Run(false); |
| 352 | return; |
| 353 | } |
| 354 | |
| 355 | // This interface should be created only once. |
| 356 | CHECK(!ocr_receiver_.is_bound()); |
| 357 | |
| 358 | ocr_receiver_.Bind(std::move(ocr_service_receiver)); |
| 359 | |
| 360 | std::move(callback).Run(true); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 361 | ocr_last_used_ = base::TimeTicks::Now(); |
Ramin Halavati | b044504d | 2022-10-24 06:36:34 | [diff] [blame] | 362 | } |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 363 | |
Ramin Halavati | f26d7d4 | 2025-03-12 16:04:46 | [diff] [blame] | 364 | void ScreenAIService::BindShutdownHandler( |
| 365 | mojo::PendingRemote<mojom::ScreenAIServiceShutdownHandler> |
| 366 | shutdown_handler) { |
| 367 | DCHECK(!screen_ai_shutdown_handler_.is_bound()); |
| 368 | screen_ai_shutdown_handler_.Bind(std::move(shutdown_handler)); |
| 369 | } |
| 370 | |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 371 | void ScreenAIService::BindAnnotator( |
| 372 | mojo::PendingReceiver<mojom::ScreenAIAnnotator> annotator) { |
| 373 | screen_ai_annotators_.Add(this, std::move(annotator)); |
| 374 | } |
| 375 | |
Ramin Halavati | eddadb6 | 2022-05-04 17:29:49 | [diff] [blame] | 376 | void ScreenAIService::BindMainContentExtractor( |
| 377 | mojo::PendingReceiver<mojom::Screen2xMainContentExtractor> |
| 378 | main_content_extractor) { |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 379 | screen2x_main_content_extractors_.Add(this, |
| 380 | std::move(main_content_extractor)); |
Ramin Halavati | eddadb6 | 2022-05-04 17:29:49 | [diff] [blame] | 381 | } |
| 382 | |
Arthur Sonzogni | c571efb | 2024-01-26 20:26:18 | [diff] [blame] | 383 | std::optional<chrome_screen_ai::VisualAnnotation> |
Ramin Halavati | 7716b03 | 2024-08-19 14:56:20 | [diff] [blame] | 384 | ScreenAIService::PerformOcrAndRecordMetrics(const SkBitmap& image) { |
Ramin Halavati | 2667959 | 2025-03-13 07:24:57 | [diff] [blame] | 385 | static crash_reporter::CrashKeyString<12> memory_perform_ocr( |
| 386 | "screen_ai_mem_ocr_perform"); |
| 387 | memory_perform_ocr.Set(GetMemoryStatusForCrashKey()); |
| 388 | |
Ramin Halavati | 7716b03 | 2024-08-19 14:56:20 | [diff] [blame] | 389 | CHECK(base::Contains(ocr_client_types_, |
| 390 | screen_ai_annotators_.current_receiver())); |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 391 | OcrClientTypeForMetrics client_type = GetClientType( |
| 392 | ocr_client_types_.find(screen_ai_annotators_.current_receiver())->second); |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 393 | base::UmaHistogramEnumeration("Accessibility.ScreenAI.OCR.ClientType", |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 394 | client_type); |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 395 | |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 396 | ocr_last_used_ = base::TimeTicks::Now(); |
Ramin Halavati | 7252cd0 | 2023-05-10 07:21:33 | [diff] [blame] | 397 | auto result = library_->PerformOcr(image); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 398 | base::TimeDelta elapsed_time = base::TimeTicks::Now() - ocr_last_used_; |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 399 | int lines_count = result ? result->lines_size() : 0; |
Ramin Halavati | 3ae33ca | 2024-01-01 12:58:39 | [diff] [blame] | 400 | unsigned image_size = image.width() * image.height(); |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 401 | VLOG(1) << "OCR returned " << lines_count << " lines in " << elapsed_time; |
Ramin Halavati | 7252cd0 | 2023-05-10 07:21:33 | [diff] [blame] | 402 | |
Ramin Halavati | 18aba87 | 2024-08-23 14:33:05 | [diff] [blame] | 403 | if (!result) { |
| 404 | base::UmaHistogramEnumeration( |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 405 | "Accessibility.ScreenAI.OCR.Failed.ClientType", client_type); |
Ramin Halavati | 18aba87 | 2024-08-23 14:33:05 | [diff] [blame] | 406 | } |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 407 | if (image_size >= kLargestOcrResolution) { |
| 408 | base::UmaHistogramEnumeration( |
| 409 | "Accessibility.ScreenAI.OCR.Oversize.ClientType", client_type); |
| 410 | } |
| 411 | |
Ramin Halavati | 81b6d92 | 2024-06-07 14:20:34 | [diff] [blame] | 412 | base::UmaHistogramBoolean("Accessibility.ScreenAI.OCR.Successful", |
| 413 | result.has_value()); |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 414 | base::UmaHistogramCounts100("Accessibility.ScreenAI.OCR.LinesCount", |
| 415 | lines_count); |
Ramin Halavati | a0548b26 | 2023-06-27 05:56:47 | [diff] [blame] | 416 | base::UmaHistogramCounts10M("Accessibility.ScreenAI.OCR.ImageSize10M", |
Ramin Halavati | 3ae33ca | 2024-01-01 12:58:39 | [diff] [blame] | 417 | image_size); |
| 418 | if (image_size < 500 * 500) { |
| 419 | base::UmaHistogramTimes("Accessibility.ScreenAI.OCR.Latency.Small", |
| 420 | elapsed_time); |
| 421 | } else if (image_size < 1000 * 1000) { |
| 422 | base::UmaHistogramTimes("Accessibility.ScreenAI.OCR.Latency.Medium", |
| 423 | elapsed_time); |
| 424 | } else if (image_size < 2000 * 2000) { |
| 425 | base::UmaHistogramTimes("Accessibility.ScreenAI.OCR.Latency.Large", |
| 426 | elapsed_time); |
| 427 | } else { |
| 428 | base::UmaHistogramTimes("Accessibility.ScreenAI.OCR.Latency.XLarge", |
| 429 | elapsed_time); |
| 430 | } |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 431 | |
Ramin Halavati | 7716b03 | 2024-08-19 14:56:20 | [diff] [blame] | 432 | // MediaApp provides OCR for ChromeOS PDF viewer. |
Ramin Halavati | 4805c88 | 2025-03-03 18:28:48 | [diff] [blame] | 433 | if (client_type == OcrClientTypeForMetrics::kPdfViewer || |
| 434 | client_type == OcrClientTypeForMetrics::kMediaApp) { |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 435 | base::UmaHistogramCounts100("Accessibility.ScreenAI.OCR.LinesCount.PDF", |
| 436 | lines_count); |
| 437 | base::UmaHistogramTimes("Accessibility.ScreenAI.OCR.Time.PDF", |
| 438 | elapsed_time); |
Ramin Halavati | 696b8d5c | 2025-02-28 16:41:58 | [diff] [blame] | 439 | base::UmaHistogramCounts10M( |
| 440 | lines_count ? "Accessibility.ScreenAI.OCR.ImageSize.PDF.WithText" |
| 441 | : "Accessibility.ScreenAI.OCR.ImageSize.PDF.NoText", |
| 442 | image_size); |
Ramin Halavati | 4c47eb8 | 2024-12-03 19:06:42 | [diff] [blame] | 443 | |
Ramin Halavati | 01267d1 | 2025-01-21 18:11:30 | [diff] [blame] | 444 | if (result.has_value()) { |
| 445 | std::optional<uint64_t> most_detected_language = |
| 446 | GetMostDetectedLanguageInOcrData(*result); |
| 447 | if (most_detected_language.has_value()) { |
| 448 | base::UmaHistogramSparse( |
| 449 | "Accessibility.ScreenAI.OCR.MostDetectedLanguage.PDF", |
| 450 | most_detected_language.value()); |
| 451 | } |
Ramin Halavati | 4c47eb8 | 2024-12-03 19:06:42 | [diff] [blame] | 452 | } |
Ramin Halavati | 2650ebe | 2023-11-27 19:56:35 | [diff] [blame] | 453 | } |
| 454 | |
Ramin Halavati | 7252cd0 | 2023-05-10 07:21:33 | [diff] [blame] | 455 | return result; |
| 456 | } |
| 457 | |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 458 | void ScreenAIService::SetClientType(mojom::OcrClientType client_type) { |
| 459 | ocr_client_types_[screen_ai_annotators_.current_receiver()] = client_type; |
| 460 | } |
| 461 | |
Ramin Halavati | d0d6231b | 2025-03-20 05:36:43 | [diff] [blame] | 462 | void ScreenAIService::SetClientType(mojom::MceClientType client_type) { |
| 463 | mce_client_types_[screen2x_main_content_extractors_.current_receiver()] = |
| 464 | client_type; |
| 465 | } |
| 466 | |
Dmitry Grebenyuk | c49b95ee | 2023-05-02 05:52:07 | [diff] [blame] | 467 | void ScreenAIService::PerformOcrAndReturnAnnotation( |
| 468 | const SkBitmap& image, |
| 469 | PerformOcrAndReturnAnnotationCallback callback) { |
Arthur Sonzogni | c571efb | 2024-01-26 20:26:18 | [diff] [blame] | 470 | std::optional<chrome_screen_ai::VisualAnnotation> annotation_proto = |
Ramin Halavati | 7716b03 | 2024-08-19 14:56:20 | [diff] [blame] | 471 | PerformOcrAndRecordMetrics(image); |
Dmitry Grebenyuk | c49b95ee | 2023-05-02 05:52:07 | [diff] [blame] | 472 | |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 473 | if (annotation_proto) { |
| 474 | std::move(callback).Run(ConvertProtoToVisualAnnotation(*annotation_proto)); |
| 475 | return; |
Dmitry Grebenyuk | c49b95ee | 2023-05-02 05:52:07 | [diff] [blame] | 476 | } |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 477 | |
| 478 | std::move(callback).Run(mojom::VisualAnnotation::New()); |
Dmitry Grebenyuk | c49b95ee | 2023-05-02 05:52:07 | [diff] [blame] | 479 | } |
| 480 | |
Kyungjun Lee | 9046152 | 2023-04-25 06:50:09 | [diff] [blame] | 481 | void ScreenAIService::PerformOcrAndReturnAXTreeUpdate( |
Ramin Halavati | b446c02 | 2023-03-29 15:36:36 | [diff] [blame] | 482 | const SkBitmap& image, |
Kyungjun Lee | 9046152 | 2023-04-25 06:50:09 | [diff] [blame] | 483 | PerformOcrAndReturnAXTreeUpdateCallback callback) { |
Arthur Sonzogni | c571efb | 2024-01-26 20:26:18 | [diff] [blame] | 484 | std::optional<chrome_screen_ai::VisualAnnotation> annotation_proto = |
Ramin Halavati | 7716b03 | 2024-08-19 14:56:20 | [diff] [blame] | 485 | PerformOcrAndRecordMetrics(image); |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 486 | ui::AXTreeUpdate update = ConvertVisualAnnotationToTreeUpdate( |
| 487 | annotation_proto, gfx::Rect(image.width(), image.height())); |
Kyungjun Lee | 9046152 | 2023-04-25 06:50:09 | [diff] [blame] | 488 | |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 489 | // The original caller is always replied to, and an empty AXTreeUpdate tells |
| 490 | // that the annotation function was not successful. |
| 491 | std::move(callback).Run(update); |
Ramin Halavati | b446c02 | 2023-03-29 15:36:36 | [diff] [blame] | 492 | } |
| 493 | |
| 494 | void ScreenAIService::ExtractMainContent(const ui::AXTreeUpdate& snapshot, |
Ramin Halavati | b446c02 | 2023-03-29 15:36:36 | [diff] [blame] | 495 | ExtractMainContentCallback callback) { |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 496 | ui::AXTree tree; |
| 497 | std::optional<std::vector<int32_t>> content_node_ids; |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 498 | bool success = ExtractMainContentInternalAndRecordMetrics(snapshot, tree, |
| 499 | content_node_ids); |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 500 | |
| 501 | if (success) { |
| 502 | std::move(callback).Run(*content_node_ids); |
| 503 | } else { |
| 504 | std::move(callback).Run(std::vector<int32_t>()); |
| 505 | } |
| 506 | } |
| 507 | |
| 508 | void ScreenAIService::ExtractMainNode(const ui::AXTreeUpdate& snapshot, |
| 509 | ExtractMainNodeCallback callback) { |
| 510 | ui::AXTree tree; |
| 511 | std::optional<std::vector<int32_t>> content_node_ids; |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 512 | bool success = ExtractMainContentInternalAndRecordMetrics(snapshot, tree, |
| 513 | content_node_ids); |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 514 | |
| 515 | if (success) { |
| 516 | ui::AXNodeID main_node_id = ComputeMainNode(&tree, *content_node_ids); |
| 517 | std::move(callback).Run(main_node_id); |
| 518 | } else { |
| 519 | std::move(callback).Run(ui::kInvalidAXNodeID); |
| 520 | } |
| 521 | } |
| 522 | |
Mark Schillaci | 79c8281 | 2025-03-07 21:46:37 | [diff] [blame] | 523 | void ScreenAIService::IdentifyMainNode(const ui::AXTreeUpdate& snapshot, |
| 524 | IdentifyMainNodeCallback callback) { |
| 525 | ui::AXTree tree; |
| 526 | std::optional<std::vector<int32_t>> content_node_ids; |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 527 | bool success = ExtractMainContentInternalAndRecordMetrics(snapshot, tree, |
| 528 | content_node_ids); |
Mark Schillaci | 79c8281 | 2025-03-07 21:46:37 | [diff] [blame] | 529 | |
| 530 | if (success) { |
| 531 | ui::AXNodeID main_node_id = ComputeMainNode(&tree, *content_node_ids); |
| 532 | std::move(callback).Run(tree.GetAXTreeID(), main_node_id); |
| 533 | } else { |
| 534 | std::move(callback).Run(ui::AXTreeIDUnknown(), ui::kInvalidAXNodeID); |
| 535 | } |
| 536 | } |
| 537 | |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 538 | bool ScreenAIService::ExtractMainContentInternalAndRecordMetrics( |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 539 | const ui::AXTreeUpdate& snapshot, |
| 540 | ui::AXTree& tree, |
| 541 | std::optional<std::vector<int32_t>>& content_node_ids) { |
Ramin Halavati | d0d6231b | 2025-03-20 05:36:43 | [diff] [blame] | 542 | CHECK(base::Contains(mce_client_types_, |
| 543 | screen2x_main_content_extractors_.current_receiver())); |
| 544 | main_content_extraction_last_used_ = base::TimeTicks::Now(); |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 545 | MainContentExtractionClientTypeForMetrics client_type = GetClientType( |
| 546 | mce_client_types_[screen2x_main_content_extractors_.current_receiver()]); |
| 547 | |
| 548 | static crash_reporter::CrashKeyString<2> cpu_info( |
| 549 | "main_content_extraction_client"); |
| 550 | cpu_info.Set(base::StringPrintf("%i", static_cast<int>(client_type))); |
| 551 | |
Ramin Halavati | b446c02 | 2023-03-29 15:36:36 | [diff] [blame] | 552 | // Early return if input is empty. |
| 553 | if (snapshot.nodes.empty()) { |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 554 | base::UmaHistogramEnumeration( |
| 555 | "Accessibility.ScreenAI.MainContentExtraction.Error.SnapshotEmpty", |
| 556 | client_type); |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 557 | return false; |
Ramin Halavati | b33fc0c | 2022-05-06 09:32:22 | [diff] [blame] | 558 | } |
Ramin Halavati | e2549a9 | 2022-08-02 07:43:19 | [diff] [blame] | 559 | |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 560 | // Deserialize the snapshot and reserialize it to a view hierarchy proto. |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 561 | if (!tree.Unserialize(snapshot)) { |
| 562 | base::UmaHistogramEnumeration( |
| 563 | "Accessibility.ScreenAI.MainContentExtraction.Error." |
| 564 | "SnapshotUnserialize", |
| 565 | client_type); |
Ramin Halavati | 62e034d | 2024-08-02 08:58:22 | [diff] [blame] | 566 | return false; |
| 567 | } |
| 568 | |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 569 | std::optional<ViewHierarchyAndTreeSize> converted_snapshot = |
| 570 | SnapshotToViewHierarchy(tree); |
| 571 | if (!converted_snapshot) { |
| 572 | base::UmaHistogramEnumeration( |
| 573 | "Accessibility.ScreenAI.MainContentExtraction.Error.SnapshotProto", |
| 574 | client_type); |
| 575 | return false; |
| 576 | } |
| 577 | |
| 578 | base::TimeTicks start_time = base::TimeTicks::Now(); |
Ramin Halavati | 62e034d | 2024-08-02 08:58:22 | [diff] [blame] | 579 | content_node_ids = |
| 580 | library_->ExtractMainContent(converted_snapshot->serialized_proto); |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 581 | base::TimeDelta elapsed_time = base::TimeTicks::Now() - start_time; |
| 582 | |
| 583 | bool successful = |
| 584 | content_node_ids.has_value() && content_node_ids->size() > 0; |
Ramin Halavati | 81b6d92 | 2024-06-07 14:20:34 | [diff] [blame] | 585 | base::UmaHistogramBoolean( |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 586 | "Accessibility.ScreenAI.MainContentExtraction.Successful2", successful); |
| 587 | |
| 588 | if (!content_node_ids.has_value()) { |
| 589 | base::UmaHistogramEnumeration( |
| 590 | "Accessibility.ScreenAI.MainContentExtraction.Error.ResultNull", |
| 591 | client_type); |
| 592 | } else if (content_node_ids->empty()) { |
| 593 | base::UmaHistogramEnumeration( |
| 594 | "Accessibility.ScreenAI.MainContentExtraction.Error.ResultEmpty", |
| 595 | client_type); |
| 596 | } |
| 597 | |
| 598 | if (successful) { |
| 599 | base::UmaHistogramTimes( |
| 600 | "Accessibility.ScreenAI.MainContentExtraction.Latency.Success", |
| 601 | elapsed_time); |
Ramin Halavati | 5aad701d | 2023-05-09 17:23:29 | [diff] [blame] | 602 | VLOG(2) << "Screen2x returned " << content_node_ids->size() << " node ids."; |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 603 | return true; |
| 604 | } else { |
Ramin Halavati | b3fe2fe | 2025-03-20 16:53:03 | [diff] [blame^] | 605 | base::UmaHistogramTimes( |
| 606 | "Accessibility.ScreenAI.MainContentExtraction.Latency.Failure", |
| 607 | elapsed_time); |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 608 | VLOG(0) << "Screen2x returned no results."; |
| 609 | return false; |
Ramin Halavati | 5aad701d | 2023-05-09 17:23:29 | [diff] [blame] | 610 | } |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 611 | } |
Ramin Halavati | 89a36c9 | 2023-05-05 15:01:03 | [diff] [blame] | 612 | |
Abigail Klein | 1867370 | 2024-03-05 20:59:06 | [diff] [blame] | 613 | ui::AXNodeID ScreenAIService::ComputeMainNodeForTesting( |
| 614 | const ui::AXTree* tree, |
| 615 | const std::vector<ui::AXNodeID>& content_node_ids) { |
| 616 | return ComputeMainNode(tree, content_node_ids); |
Abigail Klein | dfdde35 | 2023-01-27 21:03:10 | [diff] [blame] | 617 | } |
| 618 | |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 619 | void ScreenAIService::OcrReceiverDisconnected() { |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 620 | auto entry = ocr_client_types_.find(screen_ai_annotators_.current_receiver()); |
| 621 | if (entry != ocr_client_types_.end()) { |
| 622 | ocr_client_types_.erase(entry); |
| 623 | } |
Ramin Halavati | 6ff5641 | 2024-08-13 07:15:54 | [diff] [blame] | 624 | |
| 625 | CheckIdleStateAfterDelay(); |
| 626 | } |
| 627 | |
| 628 | void ScreenAIService::CheckIdleStateAfterDelay() { |
| 629 | // Check if service is idle, a little after the client disconnects. |
| 630 | base::SequencedTaskRunner::GetCurrentDefault()->PostDelayedTask( |
| 631 | FROM_HERE, |
| 632 | base::BindOnce(&ScreenAIService::ShutDownIfNoClients, |
| 633 | weak_ptr_factory_.GetWeakPtr()), |
| 634 | kCoolDownTime); |
Ramin Halavati | ad969ef | 2024-05-21 06:59:47 | [diff] [blame] | 635 | } |
| 636 | |
Ramin Halavati | 3842e7e | 2024-07-31 04:39:21 | [diff] [blame] | 637 | void ScreenAIService::ShutDownIfNoClients() { |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 638 | const base::TimeTicks kIdlenessThreshold = |
| 639 | base::TimeTicks::Now() - kIdleCheckingDelay; |
Ramin Halavati | dc0b2e51 | 2024-08-16 17:39:09 | [diff] [blame] | 640 | bool ocr_not_needed = |
| 641 | !screen_ai_annotators_.size() || ocr_last_used_ < kIdlenessThreshold; |
| 642 | bool main_content_extractioncan_not_needed = |
| 643 | !screen2x_main_content_extractors_.size() || |
| 644 | main_content_extraction_last_used_ < kIdlenessThreshold; |
| 645 | |
| 646 | if (ocr_not_needed && main_content_extractioncan_not_needed) { |
Ramin Halavati | f26d7d4 | 2025-03-12 16:04:46 | [diff] [blame] | 647 | screen_ai_shutdown_handler_->ShuttingDownOnIdle(); |
Ramin Halavati | dc0b2e51 | 2024-08-16 17:39:09 | [diff] [blame] | 648 | VLOG(2) << "Shutting down since no client or idle."; |
| 649 | base::Process::TerminateCurrentProcessImmediately(0); |
Ramin Halavati | 2acf236 | 2024-08-02 14:22:53 | [diff] [blame] | 650 | } |
Ramin Halavati | 3842e7e | 2024-07-31 04:39:21 | [diff] [blame] | 651 | } |
| 652 | |
Ramin Halavati | 26dcce2 | 2022-02-23 13:11:14 | [diff] [blame] | 653 | } // namespace screen_ai |