drive: Retry sync task with a long delay on server errors

Add a new FileError value FILE_ERROR_SERVICE_UNAVAILABLE
Queue the task again with a long delay on FILE_ERROR_SERVICE_UNAVAILABLE in SyncClient.

BUG=266643

Review URL: https://chromiumcodereview.appspot.com/22385007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@216465 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/chrome/browser/chromeos/drive/file_errors.cc b/chrome/browser/chromeos/drive/file_errors.cc
index 05d76f9..3778eae2 100644
--- a/chrome/browser/chromeos/drive/file_errors.cc
+++ b/chrome/browser/chromeos/drive/file_errors.cc
@@ -63,6 +63,9 @@
 
     case FILE_ERROR_NO_LOCAL_SPACE:
       return "FILE_ERROR_NO_LOCAL_SPACE";
+
+    case FILE_ERROR_SERVICE_UNAVAILABLE:
+      return "FILE_ERROR_SERVICE_UNAVAILABLE";
   }
 
   NOTREACHED();
@@ -124,6 +127,9 @@
 
     case FILE_ERROR_NO_LOCAL_SPACE:
       return base::PLATFORM_FILE_ERROR_FAILED;
+
+    case FILE_ERROR_SERVICE_UNAVAILABLE:
+      return base::PLATFORM_FILE_ERROR_FAILED;
   }
 
   NOTREACHED();
@@ -141,6 +147,9 @@
       return FILE_ERROR_ACCESS_DENIED;
     case google_apis::HTTP_NOT_FOUND:
       return FILE_ERROR_NOT_FOUND;
+    case google_apis::HTTP_INTERNAL_SERVER_ERROR:
+    case google_apis::HTTP_SERVICE_UNAVAILABLE:
+      return FILE_ERROR_SERVICE_UNAVAILABLE;
     case google_apis::HTTP_NOT_IMPLEMENTED:
       return FILE_ERROR_INVALID_OPERATION;
     case google_apis::GDATA_CANCELLED:
diff --git a/chrome/browser/chromeos/drive/file_errors.h b/chrome/browser/chromeos/drive/file_errors.h
index 08cf5a15..4b7bc671 100644
--- a/chrome/browser/chromeos/drive/file_errors.h
+++ b/chrome/browser/chromeos/drive/file_errors.h
@@ -30,6 +30,7 @@
   FILE_ERROR_INVALID_URL = -15,
   FILE_ERROR_NO_CONNECTION = -16,
   FILE_ERROR_NO_LOCAL_SPACE = -17,
+  FILE_ERROR_SERVICE_UNAVAILABLE = -18,
 };
 
 // Used as callbacks for file operations.
diff --git a/chrome/browser/chromeos/drive/sync_client.cc b/chrome/browser/chromeos/drive/sync_client.cc
index c1f487b0..31771d65 100644
--- a/chrome/browser/chromeos/drive/sync_client.cc
+++ b/chrome/browser/chromeos/drive/sync_client.cc
@@ -38,6 +38,9 @@
 // and shorten the delay. crbug.com/134774
 const int kDelaySeconds = 5;
 
+// The delay constant is used to delay retrying a sync task on server errors.
+const int kLongDelaySeconds = 600;
+
 // Appends |resource_id| to |to_fetch| if the file is pinned but not fetched
 // (not present locally), or to |to_upload| if the file is dirty but not
 // uploaded.
@@ -78,6 +81,7 @@
                                                          metadata,
                                                          cache)),
       delay_(base::TimeDelta::FromSeconds(kDelaySeconds)),
+      long_delay_(base::TimeDelta::FromSeconds(kLongDelaySeconds)),
       weak_ptr_factory_(this) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 }
@@ -110,7 +114,7 @@
 void SyncClient::AddFetchTask(const std::string& resource_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
-  AddTaskToQueue(FETCH, resource_id);
+  AddTaskToQueue(FETCH, resource_id, delay_);
 }
 
 void SyncClient::RemoveFetchTask(const std::string& resource_id) {
@@ -123,11 +127,12 @@
 void SyncClient::AddUploadTask(const std::string& resource_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
-  AddTaskToQueue(UPLOAD, resource_id);
+  AddTaskToQueue(UPLOAD, resource_id, delay_);
 }
 
 void SyncClient::AddTaskToQueue(SyncType type,
-                                const std::string& resource_id) {
+                                const std::string& resource_id,
+                                const base::TimeDelta& delay) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 
   // If the same task is already queued, ignore this task.
@@ -156,7 +161,7 @@
                  weak_ptr_factory_.GetWeakPtr(),
                  type,
                  resource_id),
-      delay_);
+      delay);
 }
 
 void SyncClient::StartTask(SyncType type, const std::string& resource_id) {
@@ -205,13 +210,13 @@
   for (size_t i = 0; i < to_upload->size(); ++i) {
     const std::string& resource_id = (*to_upload)[i];
     DVLOG(1) << "Queuing to upload: " << resource_id;
-    AddTaskToQueue(UPLOAD_NO_CONTENT_CHECK, resource_id);
+    AddTaskToQueue(UPLOAD_NO_CONTENT_CHECK, resource_id, delay_);
   }
 
   for (size_t i = 0; i < to_fetch->size(); ++i) {
     const std::string& resource_id = (*to_fetch)[i];
     DVLOG(1) << "Queuing to fetch: " << resource_id;
-    AddTaskToQueue(FETCH, resource_id);
+    AddTaskToQueue(FETCH, resource_id, delay_);
   }
 }
 
@@ -284,7 +289,7 @@
   }
 
   // Finally, adding to the queue.
-  AddTaskToQueue(FETCH, resource_id);
+  AddTaskToQueue(FETCH, resource_id, delay_);
 }
 
 void SyncClient::OnFetchFileComplete(const std::string& resource_id,
@@ -308,7 +313,11 @@
         break;
       case FILE_ERROR_NO_CONNECTION:
         // Re-queue the task so that we'll retry once the connection is back.
-        AddTaskToQueue(FETCH, resource_id);
+        AddTaskToQueue(FETCH, resource_id, delay_);
+        break;
+      case FILE_ERROR_SERVICE_UNAVAILABLE:
+        // Re-queue the task so that we'll retry once the service is back.
+        AddTaskToQueue(FETCH, resource_id, long_delay_);
         break;
       default:
         LOG(WARNING) << "Failed to fetch " << resource_id
@@ -329,7 +338,11 @@
     switch (error) {
       case FILE_ERROR_NO_CONNECTION:
         // Re-queue the task so that we'll retry once the connection is back.
-        AddTaskToQueue(UPLOAD_NO_CONTENT_CHECK, resource_id);
+        AddTaskToQueue(UPLOAD_NO_CONTENT_CHECK, resource_id, delay_);
+        break;
+      case FILE_ERROR_SERVICE_UNAVAILABLE:
+        // Re-queue the task so that we'll retry once the service is back.
+        AddTaskToQueue(UPLOAD_NO_CONTENT_CHECK, resource_id, long_delay_);
         break;
       default:
         LOG(WARNING) << "Failed to upload " << resource_id << ": "
diff --git a/chrome/browser/chromeos/drive/sync_client.h b/chrome/browser/chromeos/drive/sync_client.h
index f5c6734..3178f5bc 100644
--- a/chrome/browser/chromeos/drive/sync_client.h
+++ b/chrome/browser/chromeos/drive/sync_client.h
@@ -91,7 +91,9 @@
  private:
   // Adds the given task to the queue. If the same task is queued, remove the
   // existing one, and adds a new one to the end of the queue.
-  void AddTaskToQueue(SyncType type, const std::string& resource_id);
+  void AddTaskToQueue(SyncType type,
+                      const std::string& resource_id,
+                      const base::TimeDelta& delay);
 
   // Called when a task is ready to be added to the queue.
   void StartTask(SyncType type, const std::string& resource_id);
@@ -153,9 +155,12 @@
   // removed before starting, they will be cancelled.
   std::set<std::string> pending_fetch_list_;
 
-  // The delay is used for delaying processing SyncTasks in DoSyncLoop().
+  // The delay is used for delaying processing tasks in AddTaskToQueue().
   base::TimeDelta delay_;
 
+  // The delay is used for delaying retry of tasks on server errors.
+  base::TimeDelta long_delay_;
+
   // Note: This should remain the last member so it'll be destroyed and
   // invalidate its weak pointers before any other members are destroyed.
   base::WeakPtrFactory<SyncClient> weak_ptr_factory_;