Status: FAILED
Triggered by: -
Duration: 7 hours
Actions:

Definition (rev ac465d42d2d3f0b5fd3dee94a9a9d505c11cae99)

buildbucket: <
  server: "cr-buildbucket-dev.appspot.com"
  bucket: "ci"
  builder: "mac-rel-dev"
>

Incoming triggers

Incoming properties

{
  "branch": "refs/heads/main",
  "repository": "https://chromium.googlesource.com/chromium/src",
  "revision": "ba2204e71c3812528e0a4d408a227e334025c820"
}

Incoming tags

  • buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820
  • gitiles_ref:refs/heads/main

Debug log (UTC)

[12:26:31.055] New invocation is queued and will start shortly
[12:26:32.122] Starting the invocation (attempt 1)
[12:26:32.146] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:26:32.146] Popped gitiles commit info from properties and tags
[12:26:32.146] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:26:32.147] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:26:32.147] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:27:05.529] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:27:05.530] The invocation will be retried
[12:27:10.608] Starting the invocation (attempt 2)
[12:27:10.635] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:27:10.635] Popped gitiles commit info from properties and tags
[12:27:10.635] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:27:10.635] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:27:10.635] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:27:43.841] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:27:43.841] The invocation will be retried
[12:27:53.900] Starting the invocation (attempt 3)
[12:27:53.931] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:27:53.931] Popped gitiles commit info from properties and tags
[12:27:53.931] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:27:53.941] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:27:53.941] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:28:27.200] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:28:27.200] The invocation will be retried
[12:28:48.141] Starting the invocation (attempt 4)
[12:28:48.170] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:28:48.170] Popped gitiles commit info from properties and tags
[12:28:48.170] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:28:48.170] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:28:48.170] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:29:21.790] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:29:21.790] The invocation will be retried
[12:30:01.853] Starting the invocation (attempt 5)
[12:30:01.884] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:30:01.884] Popped gitiles commit info from properties and tags
[12:30:01.885] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:30:01.885] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:30:01.885] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:30:35.347] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:30:35.348] The invocation will be retried
[12:31:55.401] Starting the invocation (attempt 6)
[12:31:55.435] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:31:55.435] Popped gitiles commit info from properties and tags
[12:31:55.435] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:31:55.436] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:31:55.436] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:32:29.103] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:32:29.103] The invocation will be retried
[12:35:10.117] Starting the invocation (attempt 7)
[12:35:10.146] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:35:10.146] Popped gitiles commit info from properties and tags
[12:35:10.146] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:35:10.146] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:35:10.146] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:35:43.747] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:35:43.748] The invocation will be retried
[12:41:03.801] Starting the invocation (attempt 8)
[12:41:03.831] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:41:03.831] Popped gitiles commit info from properties and tags
[12:41:03.831] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:41:03.833] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:41:03.833] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:41:37.343] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:41:37.343] The invocation will be retried
[12:51:37.405] Starting the invocation (attempt 9)
[12:51:37.450] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[12:51:37.450] Popped gitiles commit info from properties and tags
[12:51:37.450] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[12:51:37.450] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[12:51:37.450] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[12:52:11.300] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[12:52:11.300] The invocation will be retried
[13:02:11.349] Starting the invocation (attempt 10)
[13:02:11.375] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:02:11.375] Popped gitiles commit info from properties and tags
[13:02:11.375] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:02:11.375] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:02:11.375] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:02:44.763] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:02:44.764] The invocation will be retried
[13:12:44.831] Starting the invocation (attempt 11)
[13:12:44.861] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:12:44.861] Popped gitiles commit info from properties and tags
[13:12:44.861] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:12:44.861] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:12:44.861] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:13:17.973] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:13:17.973] The invocation will be retried
[13:23:18.028] Starting the invocation (attempt 12)
[13:23:18.057] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:23:18.057] Popped gitiles commit info from properties and tags
[13:23:18.057] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:23:18.057] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:23:18.057] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:23:51.373] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:23:51.373] The invocation will be retried
[13:33:51.452] Starting the invocation (attempt 13)
[13:33:51.487] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:33:51.488] Popped gitiles commit info from properties and tags
[13:33:51.488] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:33:51.488] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:33:51.488] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:34:24.777] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:34:24.777] The invocation will be retried
[13:44:24.829] Starting the invocation (attempt 14)
[13:44:24.853] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:44:24.853] Popped gitiles commit info from properties and tags
[13:44:24.853] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:44:24.854] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:44:24.854] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:44:58.463] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:44:58.463] The invocation will be retried
[13:54:58.519] Starting the invocation (attempt 15)
[13:54:58.549] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[13:54:58.549] Popped gitiles commit info from properties and tags
[13:54:58.549] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[13:54:58.549] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[13:54:58.549] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[13:55:32.505] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[13:55:32.505] The invocation will be retried
[14:05:32.585] Starting the invocation (attempt 16)
[14:05:32.634] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:05:32.634] Popped gitiles commit info from properties and tags
[14:05:32.634] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:05:32.634] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:05:32.634] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:06:06.563] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:06:06.563] The invocation will be retried
[14:16:06.623] Starting the invocation (attempt 17)
[14:16:06.649] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:16:06.649] Popped gitiles commit info from properties and tags
[14:16:06.649] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:16:06.649] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:16:06.649] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:16:41.436] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:16:41.436] The invocation will be retried
[14:26:42.101] Starting the invocation (attempt 18)
[14:26:42.128] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:26:42.128] Popped gitiles commit info from properties and tags
[14:26:42.128] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:26:42.128] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:26:42.128] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:27:15.318] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:27:15.318] The invocation will be retried
[14:37:15.374] Starting the invocation (attempt 19)
[14:37:15.401] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:37:15.401] Popped gitiles commit info from properties and tags
[14:37:15.401] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:37:15.401] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:37:15.401] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:37:48.412] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:37:48.412] The invocation will be retried
[14:47:48.474] Starting the invocation (attempt 20)
[14:47:48.509] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:47:48.509] Popped gitiles commit info from properties and tags
[14:47:48.509] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:47:48.509] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:47:48.510] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:48:22.079] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:48:22.080] The invocation will be retried
[14:58:22.137] Starting the invocation (attempt 21)
[14:58:22.161] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[14:58:22.162] Popped gitiles commit info from properties and tags
[14:58:22.162] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[14:58:22.162] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[14:58:22.162] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[14:58:55.428] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[14:58:55.429] The invocation will be retried
[15:08:55.484] Starting the invocation (attempt 22)
[15:08:55.511] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[15:08:55.511] Popped gitiles commit info from properties and tags
[15:08:55.511] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[15:08:55.511] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[15:08:55.511] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[15:09:28.641] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[15:09:28.642] The invocation will be retried
[15:19:28.702] Starting the invocation (attempt 23)
[15:19:28.732] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[15:19:28.732] Popped gitiles commit info from properties and tags
[15:19:28.732] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[15:19:28.732] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[15:19:28.732] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[15:20:01.990] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[15:20:01.990] The invocation will be retried
[15:30:02.050] Starting the invocation (attempt 24)
[15:30:02.084] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[15:30:02.084] Popped gitiles commit info from properties and tags
[15:30:02.084] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[15:30:02.084] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[15:30:02.085] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[15:30:35.260] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[15:30:35.260] The invocation will be retried
[15:40:35.324] Starting the invocation (attempt 25)
[15:40:35.353] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[15:40:35.353] Popped gitiles commit info from properties and tags
[15:40:35.353] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[15:40:35.353] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[15:40:35.353] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[15:41:08.530] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[15:41:08.530] The invocation will be retried
[15:51:08.594] Starting the invocation (attempt 26)
[15:51:08.627] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[15:51:08.627] Popped gitiles commit info from properties and tags
[15:51:08.627] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[15:51:08.627] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[15:51:08.627] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[15:51:42.191] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[15:51:42.192] The invocation will be retried
[16:01:42.265] Starting the invocation (attempt 27)
[16:01:42.291] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:01:42.292] Popped gitiles commit info from properties and tags
[16:01:42.292] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:01:42.292] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:01:42.292] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:02:15.503] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:02:15.503] The invocation will be retried
[16:12:16.035] Starting the invocation (attempt 28)
[16:12:16.066] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:12:16.067] Popped gitiles commit info from properties and tags
[16:12:16.067] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:12:16.067] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:12:16.067] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:12:49.184] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:12:49.184] The invocation will be retried
[16:22:49.590] Starting the invocation (attempt 29)
[16:22:49.618] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:22:49.618] Popped gitiles commit info from properties and tags
[16:22:49.618] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:22:49.619] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:22:49.619] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:23:23.220] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:23:23.221] The invocation will be retried
[16:33:23.326] Starting the invocation (attempt 30)
[16:33:23.361] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:33:23.361] Popped gitiles commit info from properties and tags
[16:33:23.361] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:33:23.361] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:33:23.361] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:33:56.628] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:33:56.628] The invocation will be retried
[16:43:56.939] Starting the invocation (attempt 31)
[16:43:56.970] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:43:56.970] Popped gitiles commit info from properties and tags
[16:43:56.970] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:43:56.972] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:43:56.972] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:44:30.273] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:44:30.273] The invocation will be retried
[16:54:30.331] Starting the invocation (attempt 32)
[16:54:30.359] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[16:54:30.359] Popped gitiles commit info from properties and tags
[16:54:30.359] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[16:54:30.359] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[16:54:30.359] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[16:55:03.520] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[16:55:03.520] The invocation will be retried
[17:05:03.580] Starting the invocation (attempt 33)
[17:05:03.609] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:05:03.609] Popped gitiles commit info from properties and tags
[17:05:03.609] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:05:03.609] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:05:03.609] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:05:36.899] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:05:36.899] The invocation will be retried
[17:15:37.201] Starting the invocation (attempt 34)
[17:15:37.229] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:15:37.229] Popped gitiles commit info from properties and tags
[17:15:37.229] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:15:37.229] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:15:37.229] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:16:10.672] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:16:10.672] The invocation will be retried
[17:26:10.937] Starting the invocation (attempt 35)
[17:26:10.967] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:26:10.967] Popped gitiles commit info from properties and tags
[17:26:10.967] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:26:10.967] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:26:10.968] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:26:44.573] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:26:44.573] The invocation will be retried
[17:36:45.242] Starting the invocation (attempt 36)
[17:36:45.273] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:36:45.273] Popped gitiles commit info from properties and tags
[17:36:45.273] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:36:45.273] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:36:45.274] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:37:18.409] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:37:18.409] The invocation will be retried
[17:47:18.941] Starting the invocation (attempt 37)
[17:47:18.967] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:47:18.967] Popped gitiles commit info from properties and tags
[17:47:18.967] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:47:18.968] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:47:18.969] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:47:52.340] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:47:52.340] The invocation will be retried
[17:57:53.131] Starting the invocation (attempt 38)
[17:57:53.180] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[17:57:53.180] Popped gitiles commit info from properties and tags
[17:57:53.180] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[17:57:53.181] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[17:57:53.181] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[17:58:26.319] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[17:58:26.319] The invocation will be retried
[18:08:27.132] Starting the invocation (attempt 39)
[18:08:27.160] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[18:08:27.160] Popped gitiles commit info from properties and tags
[18:08:27.160] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[18:08:27.160] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[18:08:27.161] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[18:09:00.354] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[18:09:00.354] The invocation will be retried
[18:19:00.601] Starting the invocation (attempt 40)
[18:19:00.625] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[18:19:00.625] Popped gitiles commit info from properties and tags
[18:19:00.625] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[18:19:00.625] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[18:19:00.625] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[18:19:34.196] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[18:19:34.197] The invocation will be retried
[18:29:34.694] Starting the invocation (attempt 41)
[18:29:34.721] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[18:29:34.721] Popped gitiles commit info from properties and tags
[18:29:34.721] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[18:29:34.721] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[18:29:34.721] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[18:30:07.957] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[18:30:07.957] The invocation will be retried
[18:40:08.019] Starting the invocation (attempt 42)
[18:40:08.052] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[18:40:08.052] Popped gitiles commit info from properties and tags
[18:40:08.052] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[18:40:08.055] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[18:40:08.055] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[18:40:41.267] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[18:40:41.267] The invocation will be retried
[18:50:41.335] Starting the invocation (attempt 43)
[18:50:41.362] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[18:50:41.362] Popped gitiles commit info from properties and tags
[18:50:41.362] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[18:50:41.364] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[18:50:41.364] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[18:51:14.827] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[18:51:14.827] The invocation will be retried
[19:01:14.890] Starting the invocation (attempt 44)
[19:01:14.922] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:01:14.922] Popped gitiles commit info from properties and tags
[19:01:14.922] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:01:14.922] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:01:14.922] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:01:48.338] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:01:48.339] The invocation will be retried
[19:11:48.397] Starting the invocation (attempt 45)
[19:11:48.426] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:11:48.426] Popped gitiles commit info from properties and tags
[19:11:48.426] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:11:48.426] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:11:48.426] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:12:21.612] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:12:21.612] The invocation will be retried
[19:22:21.863] Starting the invocation (attempt 46)
[19:22:21.890] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:22:21.890] Popped gitiles commit info from properties and tags
[19:22:21.890] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:22:21.890] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:22:21.891] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:22:55.212] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:22:55.212] The invocation will be retried
[19:32:55.556] Starting the invocation (attempt 47)
[19:32:55.588] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:32:55.588] Popped gitiles commit info from properties and tags
[19:32:55.588] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:32:55.588] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:32:55.588] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:33:28.791] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:33:28.791] The invocation will be retried
[19:43:29.214] Starting the invocation (attempt 48)
[19:43:29.253] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:43:29.253] Popped gitiles commit info from properties and tags
[19:43:29.253] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:43:29.253] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:43:29.253] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:44:02.830] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:44:02.831] The invocation will be retried
[19:54:02.893] Starting the invocation (attempt 49)
[19:54:02.925] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[19:54:02.925] Popped gitiles commit info from properties and tags
[19:54:02.925] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[19:54:02.925] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[19:54:02.925] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[19:54:37.145] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[19:54:37.146] The invocation will be retried
[20:04:37.213] Starting the invocation (attempt 50)
[20:04:37.244] Popped buildset tag "buildset:commit/gitiles/chromium.googlesource.com/chromium/src/+/ba2204e71c3812528e0a4d408a227e334025c820"
[20:04:37.244] Popped gitiles commit info from properties and tags
[20:04:37.244] Preparing PubSub topic for "https://cr-buildbucket-dev.appspot.com"
[20:04:37.244] PubSub topic is "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
[20:04:37.244] Buildbucket request:
{
  "requestId": "8897719572791857360",
  "builder": {
    "project": "chromium",
    "bucket": "ci",
    "builder": "mac-rel-dev"
  },
  "properties": {
    },
  "gitilesCommit": {
    "host": "chromium.googlesource.com",
    "project": "chromium/src",
    "id": "ba2204e71c3812528e0a4d408a227e334025c820",
    "ref": "refs/heads/main"
  },
  "tags": [
    {
      "key": "scheduler_invocation_id",
      "value": "8897719572791857360"
    },
    {
      "key": "scheduler_job_id",
      "value": "chromium/mac-rel-dev"
    },
    {
      "key": "user_agent",
      "value": "luci-scheduler-dev"
    }
  ],
  "notify": {
    "pubsubTopic": "projects/luci-scheduler-dev/topics/scheduler.buildbucket.cr-buildbucket-dev~appspot.gserviceaccount.com"
  }
}
[20:05:10.789] Failed to schedule Buildbucket build - rpc error: code = Internal desc = Internal server error
[20:05:10.789] Invocation finished in 7h38m39.738688193s with status FAILED