-
-
Save kevin85421/16458127560a1c3055a4e2741ed2adeb to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"level":"info","ts":"2024-04-16T23:09:44.135Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."} | |
| {"level":"info","ts":"2024-04-16T23:09:44.135Z","logger":"setup","msg":"Setup manager"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.144Z","logger":"setup","msg":"starting manager"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.144Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.144Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false} | |
| {"level":"info","ts":"2024-04-16T23:09:44.144Z","msg":"starting server","kind":"health probe","addr":"[::]:8082"} | |
| I0416 23:09:44.244768 1 leaderelection.go:250] attempting to acquire leader lease default/ray-operator-leader... | |
| I0416 23:09:44.250944 1 leaderelection.go:260] successfully acquired lease default/ray-operator-leader | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayCluster","msg":"Starting Controller"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Ingress"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayService","msg":"Starting Controller"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.251Z","logger":"controllers.RayJob","msg":"Starting Controller"} | |
| {"level":"info","ts":"2024-04-16T23:09:44.753Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1} | |
| {"level":"info","ts":"2024-04-16T23:09:44.753Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1} | |
| {"level":"info","ts":"2024-04-16T23:09:44.754Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1} | |
| {"level":"info","ts":"2024-04-16T23:10:43.121Z","logger":"controllers.RayService","msg":"No active Ray cluster. RayService operator should prepare a new Ray cluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5ff1b419-60b1-46ca-ae88-f39c408cd874"} | |
| {"level":"info","ts":"2024-04-16T23:10:43.121Z","logger":"controllers.RayService","msg":"Current cluster is unhealthy, prepare to restart.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5ff1b419-60b1-46ca-ae88-f39c408cd874","Status":{"activeServiceStatus":{"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"pendingServiceStatus":{"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"observedGeneration":1}} | |
| {"level":"info","ts":"2024-04-16T23:10:43.129Z","logger":"KubeAPIWarningLogger","msg":"unknown field \"spec.rayClusterConfig.headGroupSpec.template.metadata.creationTimestamp\""} | |
| {"level":"info","ts":"2024-04-16T23:10:43.130Z","logger":"controllers.RayService","msg":"Done reconcileRayCluster update status, enter next loop to create new ray cluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5ff1b419-60b1-46ca-ae88-f39c408cd874"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.131Z","logger":"controllers.RayService","msg":"Creating a new pending RayCluster instance.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.131Z","logger":"controllers.RayService","msg":"createRayClusterInstance","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69","rayClusterInstanceName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.131Z","logger":"controllers.RayService","msg":"No pending RayCluster, creating RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.137Z","logger":"KubeAPIWarningLogger","msg":"unknown field \"spec.headGroupSpec.template.metadata.creationTimestamp\""} | |
| {"level":"info","ts":"2024-04-16T23:10:45.137Z","logger":"controllers.RayService","msg":"created rayCluster for rayService","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69","rayCluster":{"namespace":"default","name":"rayservice-sample-raycluster-gt2bm"}} | |
| {"level":"info","ts":"2024-04-16T23:10:45.137Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"error","ts":"2024-04-16T23:10:45.137Z","logger":"controllers.RayService","msg":"Failed to check if head Pod is running and ready!","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69","error":"Found 0 head pods for RayCluster rayservice-sample-raycluster-gt2bm in the namespace default","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).reconcileServe\n\t/workspace/controllers/ray/rayservice_controller.go:1069\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"error","ts":"2024-04-16T23:10:45.137Z","logger":"controllers.RayService","msg":"Fail to reconcileServe.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4cc713ce-e30e-47a7-a70a-606189c78c69","error":"Found 0 head pods for RayCluster rayservice-sample-raycluster-gt2bm in the namespace default","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:169\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.137Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.138Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"716b87b6-b27f-4f49-8502-f925759e6e2d","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"error","ts":"2024-04-16T23:10:45.138Z","logger":"controllers.RayService","msg":"Failed to check if head Pod is running and ready!","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"716b87b6-b27f-4f49-8502-f925759e6e2d","error":"Found 0 head pods for RayCluster rayservice-sample-raycluster-gt2bm in the namespace default","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).reconcileServe\n\t/workspace/controllers/ray/rayservice_controller.go:1069\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:168\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"error","ts":"2024-04-16T23:10:45.138Z","logger":"controllers.RayService","msg":"Fail to reconcileServe.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"716b87b6-b27f-4f49-8502-f925759e6e2d","error":"Found 0 head pods for RayCluster rayservice-sample-raycluster-gt2bm in the namespace default","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:169\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"Pod Service created successfully","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","service name":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","Found 0 head Pods; creating a head Pod for the RayCluster.":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"rayservice-sample-raycluster-gt2bm","ray.io/group":"headgroup","ray.io/identifier":"rayservice-sample-raycluster-gt2bm-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"2Gi"},"requests":{"cpu":"2","memory":"2Gi"}}} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","rayStartCmd":"ray start --head --memory=2147483648 --dashboard-host=0.0.0.0 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 --num-cpus=2 "} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head --memory=2147483648 --dashboard-host=0.0.0.0 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 --num-cpus=2 "} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","enabled":true} | |
| {"level":"info","ts":"2024-04-16T23:10:45.145Z","logger":"controllers.RayCluster","msg":"createHeadPod","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","head pod with name":"rayservice-sample-raycluster-gt2bm-head-"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running; Pod Name: rayservice-sample-raycluster-gt2bm-head-9724b; Pod Status.Phase: Pending","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.153Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","detect inconsistency":"old Endpoints: map[], new Endpoints: map[client:10001 dashboard:8265 gcs-server:6379 metrics:8080 serve:8000], old Head: { }, new Head: { 10.96.11.81}"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.153Z","logger":"controllers.RayCluster","msg":"rayClusterReconcile","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","Update CR status":"rayservice-sample-raycluster-gt2bm","status":{"desiredCPU":"2","desiredMemory":"2Gi","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2024-04-16T23:10:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceIP":"10.96.11.81"},"observedGeneration":1}} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"d9c8443b-9b58-4d61-95aa-fa4f137c30ca","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Pending","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Pending, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"796e210e-17aa-44e2-a000-dbfc7098308e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"796e210e-17aa-44e2-a000-dbfc7098308e"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"796e210e-17aa-44e2-a000-dbfc7098308e"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running; Pod Name: rayservice-sample-raycluster-gt2bm-head-9724b; Pod Status.Phase: Pending","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.161Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"40aaae12-fd5c-4ea7-8032-4c35256d466c","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Pending","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Pending, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running; Pod Name: rayservice-sample-raycluster-gt2bm-head-9724b; Pod Status.Phase: Pending","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:45.170Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"95bda105-8d4c-447e-b40e-a39c095334c4","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:10:47.138Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2910d60c-50c3-426f-819c-9fc3720e0fb2","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:47.138Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2910d60c-50c3-426f-819c-9fc3720e0fb2"} | |
| {"level":"info","ts":"2024-04-16T23:10:47.138Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2910d60c-50c3-426f-819c-9fc3720e0fb2"} | |
| {"level":"info","ts":"2024-04-16T23:10:49.139Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0184c9f4-a2fd-448c-891d-97825b034af4","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:49.139Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0184c9f4-a2fd-448c-891d-97825b034af4"} | |
| {"level":"info","ts":"2024-04-16T23:10:49.139Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0184c9f4-a2fd-448c-891d-97825b034af4"} | |
| {"level":"info","ts":"2024-04-16T23:10:51.140Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4df34b22-c89a-4a33-a2f5-b410b28456ed","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:51.140Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4df34b22-c89a-4a33-a2f5-b410b28456ed"} | |
| {"level":"info","ts":"2024-04-16T23:10:51.140Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4df34b22-c89a-4a33-a2f5-b410b28456ed"} | |
| {"level":"info","ts":"2024-04-16T23:10:53.141Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fba2b32e-6152-4d4d-b7ec-cd09e261af57","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:53.141Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fba2b32e-6152-4d4d-b7ec-cd09e261af57"} | |
| {"level":"info","ts":"2024-04-16T23:10:53.141Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fba2b32e-6152-4d4d-b7ec-cd09e261af57"} | |
| {"level":"info","ts":"2024-04-16T23:10:55.142Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cab2c69f-8c82-40f6-b0ba-de7219d81371","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:55.142Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cab2c69f-8c82-40f6-b0ba-de7219d81371"} | |
| {"level":"info","ts":"2024-04-16T23:10:55.142Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cab2c69f-8c82-40f6-b0ba-de7219d81371"} | |
| {"level":"info","ts":"2024-04-16T23:10:57.143Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6ad0f7ee-9006-4fa8-ba1d-885937fc6d16","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:57.143Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6ad0f7ee-9006-4fa8-ba1d-885937fc6d16"} | |
| {"level":"info","ts":"2024-04-16T23:10:57.143Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6ad0f7ee-9006-4fa8-ba1d-885937fc6d16"} | |
| {"level":"info","ts":"2024-04-16T23:10:59.144Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac0cf470-162f-4909-8efc-2429605d993f","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:10:59.144Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac0cf470-162f-4909-8efc-2429605d993f"} | |
| {"level":"info","ts":"2024-04-16T23:10:59.144Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac0cf470-162f-4909-8efc-2429605d993f"} | |
| {"level":"info","ts":"2024-04-16T23:11:01.146Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9167dc66-190a-4e46-8d70-b046b30d431f","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:01.146Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9167dc66-190a-4e46-8d70-b046b30d431f"} | |
| {"level":"info","ts":"2024-04-16T23:11:01.146Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9167dc66-190a-4e46-8d70-b046b30d431f"} | |
| {"level":"info","ts":"2024-04-16T23:11:03.146Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9da881b0-649b-4522-9f82-77344674dc2a","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:03.146Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9da881b0-649b-4522-9f82-77344674dc2a"} | |
| {"level":"info","ts":"2024-04-16T23:11:03.146Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"9da881b0-649b-4522-9f82-77344674dc2a"} | |
| {"level":"info","ts":"2024-04-16T23:11:05.147Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d24f96f2-12a3-466d-b2e9-5a349f26e74a","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:05.147Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d24f96f2-12a3-466d-b2e9-5a349f26e74a"} | |
| {"level":"info","ts":"2024-04-16T23:11:05.147Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d24f96f2-12a3-466d-b2e9-5a349f26e74a"} | |
| {"level":"info","ts":"2024-04-16T23:11:07.148Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f34f4a59-1973-4b7e-bc5c-7cf9a157a470","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:07.148Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f34f4a59-1973-4b7e-bc5c-7cf9a157a470"} | |
| {"level":"info","ts":"2024-04-16T23:11:07.148Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f34f4a59-1973-4b7e-bc5c-7cf9a157a470"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Running","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Running, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready; Pod Name: rayservice-sample-raycluster-gt2bm-head-9724b; Pod Status.Conditions[PodReady]: {Ready False 0001-01-01 00:00:00 +0000 UTC 2024-04-16 23:10:45 +0000 UTC ContainersNotReady containers with unready status: [ray-head]}","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","detect inconsistency":"old Endpoints: map[client:10001 dashboard:8265 gcs-server:6379 metrics:8080 serve:8000], new Endpoints: map[client:10001 dashboard:8265 gcs-server:6379 metrics:8080 serve:8000], old Head: { 10.96.11.81}, new Head: {10.244.0.6 10.96.11.81}"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.126Z","logger":"controllers.RayCluster","msg":"rayClusterReconcile","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","Update CR status":"rayservice-sample-raycluster-gt2bm","status":{"desiredCPU":"2","desiredMemory":"2Gi","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2024-04-16T23:11:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.244.0.6","serviceIP":"10.96.11.81"},"observedGeneration":1}} | |
| {"level":"info","ts":"2024-04-16T23:11:08.136Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.136Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f5f35b02-1f9f-48ed-8cd5-04f1665ea270","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:11:08.136Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8a60df2d-b69b-41d2-8d99-6ea9b68b51cd","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.136Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8a60df2d-b69b-41d2-8d99-6ea9b68b51cd"} | |
| {"level":"info","ts":"2024-04-16T23:11:08.136Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8a60df2d-b69b-41d2-8d99-6ea9b68b51cd"} | |
| {"level":"info","ts":"2024-04-16T23:11:09.150Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"a0696bf8-a3a2-4810-8c4f-8fd284f0f955","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:09.150Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"a0696bf8-a3a2-4810-8c4f-8fd284f0f955"} | |
| {"level":"info","ts":"2024-04-16T23:11:09.150Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"a0696bf8-a3a2-4810-8c4f-8fd284f0f955"} | |
| {"level":"info","ts":"2024-04-16T23:11:11.150Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"271db259-30d5-4c70-b5d8-3feeacbdaebc","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:11.150Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"271db259-30d5-4c70-b5d8-3feeacbdaebc"} | |
| {"level":"info","ts":"2024-04-16T23:11:11.150Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"271db259-30d5-4c70-b5d8-3feeacbdaebc"} | |
| {"level":"info","ts":"2024-04-16T23:11:13.152Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d7739c2f-3663-4a8a-b594-c98bf87440d4","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:13.152Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d7739c2f-3663-4a8a-b594-c98bf87440d4"} | |
| {"level":"info","ts":"2024-04-16T23:11:13.152Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d7739c2f-3663-4a8a-b594-c98bf87440d4"} | |
| {"level":"info","ts":"2024-04-16T23:11:15.152Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3e8b5235-f305-4e76-99ff-37a71b78427d","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:15.152Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3e8b5235-f305-4e76-99ff-37a71b78427d"} | |
| {"level":"info","ts":"2024-04-16T23:11:15.152Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3e8b5235-f305-4e76-99ff-37a71b78427d"} | |
| {"level":"info","ts":"2024-04-16T23:11:17.153Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7cf7ec6c-0234-41bd-9cac-1334e5f61b8f","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:17.153Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7cf7ec6c-0234-41bd-9cac-1334e5f61b8f"} | |
| {"level":"info","ts":"2024-04-16T23:11:17.153Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7cf7ec6c-0234-41bd-9cac-1334e5f61b8f"} | |
| {"level":"info","ts":"2024-04-16T23:11:19.154Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"66fabc5f-6273-4edb-bf36-f9d94f0d49ba","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:19.154Z","logger":"controllers.RayService","msg":"Skipping the update of Serve deployments because the Ray head Pod is not ready.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"66fabc5f-6273-4edb-bf36-f9d94f0d49ba"} | |
| {"level":"info","ts":"2024-04-16T23:11:19.155Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"66fabc5f-6273-4edb-bf36-f9d94f0d49ba"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Running","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Running, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","detect inconsistency":"old State: , new State: ready, old Reason: , new Reason: "} | |
| {"level":"info","ts":"2024-04-16T23:11:20.585Z","logger":"controllers.RayCluster","msg":"rayClusterReconcile","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","Update CR status":"rayservice-sample-raycluster-gt2bm","status":{"state":"ready","desiredCPU":"2","desiredMemory":"2Gi","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2024-04-16T23:11:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.244.0.6","serviceIP":"10.96.11.81"},"observedGeneration":1}} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"1ca2d5e7-4f36-4ea9-9067-3101f9b81b7d","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","shouldUpdateServe":true,"reason":"Nothing has been cached for cluster rayservice-sample-raycluster-gt2bm with key default/rayservice-sample/rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.594Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","V2 config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:20.595Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","MULTI_APP json config":"{\"applications\":[{\"deployments\":[{\"name\":\"MangoStand\",\"num_replicas\":2,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":3}},{\"name\":\"OrangeStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":2}},{\"name\":\"PearStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":1}},{\"name\":\"FruitMarket\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1}}],\"import_path\":\"fruit.deployment_graph\",\"name\":\"fruit_app\",\"route_prefix\":\"/fruit\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}},{\"deployments\":[{\"name\":\"Adder\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"increment\":3}},{\"name\":\"Multiplier\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"factor\":5}},{\"name\":\"Router\",\"num_replicas\":1}],\"import_path\":\"conditional_dag.serve_dag\",\"name\":\"math_app\",\"route_prefix\":\"/calc\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}}]}"} | |
| {"level":"error","ts":"2024-04-16T23:11:20.607Z","logger":"controllers.RayService","msg":"Fail to reconcileServe.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c0affb54-c16c-45a5-86d3-fea7e1435f50","error":"Fail to create / update Serve applications. If you observe this error consistently, please check \"Issue 5: Fail to create / update Serve applications.\" in https://docs.ray.io/en/master/cluster/kubernetes/troubleshooting/rayservice-troubleshooting.html#kuberay-raysvc-troubleshoot for more details. err: Put \"http://rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265/api/serve/applications/\": dial tcp 10.96.11.81:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:169\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.155Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.155Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.155Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.155Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","shouldUpdateServe":true,"reason":"Nothing has been cached for cluster rayservice-sample-raycluster-gt2bm with key default/rayservice-sample/rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.155Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","V2 config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:21.156Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","MULTI_APP json config":"{\"applications\":[{\"deployments\":[{\"name\":\"MangoStand\",\"num_replicas\":2,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":3}},{\"name\":\"OrangeStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":2}},{\"name\":\"PearStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":1}},{\"name\":\"FruitMarket\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1}}],\"import_path\":\"fruit.deployment_graph\",\"name\":\"fruit_app\",\"route_prefix\":\"/fruit\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}},{\"deployments\":[{\"name\":\"Adder\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"increment\":3}},{\"name\":\"Multiplier\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"factor\":5}},{\"name\":\"Router\",\"num_replicas\":1}],\"import_path\":\"conditional_dag.serve_dag\",\"name\":\"math_app\",\"route_prefix\":\"/calc\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}}]}"} | |
| {"level":"error","ts":"2024-04-16T23:11:23.162Z","logger":"controllers.RayService","msg":"Fail to reconcileServe.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c46c1bc2-6640-47f8-a15c-26052f2be52e","error":"Fail to create / update Serve applications. If you observe this error consistently, please check \"Issue 5: Fail to create / update Serve applications.\" in https://docs.ray.io/en/master/cluster/kubernetes/troubleshooting/rayservice-troubleshooting.html#kuberay-raysvc-troubleshoot for more details. err: Put \"http://rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265/api/serve/applications/\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayServiceReconciler).Reconcile\n\t/workspace/controllers/ray/rayservice_controller.go:169\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","shouldUpdateServe":true,"reason":"Nothing has been cached for cluster rayservice-sample-raycluster-gt2bm with key default/rayservice-sample/rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","V2 config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.163Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","MULTI_APP json config":"{\"applications\":[{\"deployments\":[{\"name\":\"MangoStand\",\"num_replicas\":2,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":3}},{\"name\":\"OrangeStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":2}},{\"name\":\"PearStand\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"price\":1}},{\"name\":\"FruitMarket\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1}}],\"import_path\":\"fruit.deployment_graph\",\"name\":\"fruit_app\",\"route_prefix\":\"/fruit\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}},{\"deployments\":[{\"name\":\"Adder\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"increment\":3}},{\"name\":\"Multiplier\",\"num_replicas\":1,\"ray_actor_options\":{\"num_cpus\":0.1},\"user_config\":{\"factor\":5}},{\"name\":\"Router\",\"num_replicas\":1}],\"import_path\":\"conditional_dag.serve_dag\",\"name\":\"math_app\",\"route_prefix\":\"/calc\",\"runtime_env\":{\"working_dir\":\"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"}}]}"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.178Z","logger":"controllers.RayService","msg":"updateServeDeployment","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","message":"Cached Serve config for Ray cluster rayservice-sample-raycluster-gt2bm with key default/rayservice-sample/rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:25.183Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","prev statuses":null,"serve statuses":{"fruit_app":{"name":"fruit_app","status":"DEPLOYING","deployments":{}},"math_app":{"name":"math_app","status":"DEPLOYING","deployments":{}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:25.183Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","new statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:25Z"},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:25Z"}}} | |
| {"level":"info","ts":"2024-04-16T23:11:25.183Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","isReady":false,"isActive":false} | |
| {"level":"info","ts":"2024-04-16T23:11:25.446Z","logger":"controllers.RayService","msg":"Mark cluster as waiting for Serve deployments","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3","rayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"default","name":"rayservice-sample-raycluster-gt2bm"}} | |
| {"level":"info","ts":"2024-04-16T23:11:25.446Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cf842b21-a8be-4519-9d68-1b815c4114a3"} | |
| {"level":"info","ts":"2024-04-16T23:11:27.447Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:27.447Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:27.447Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:27.447Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:27.455Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","prev statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:25Z"},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:25Z"}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"DEPLOYING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"UPDATING"},"MangoStand":{"name":"MangoStand","status":"UPDATING"},"OrangeStand":{"name":"OrangeStand","status":"UPDATING"},"PearStand":{"name":"PearStand","status":"UPDATING"}}},"math_app":{"name":"math_app","status":"DEPLOYING","deployments":{"Adder":{"name":"Adder","status":"UPDATING"},"Multiplier":{"name":"Multiplier","status":"UPDATING"},"Router":{"name":"Router","status":"UPDATING"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:27.455Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","new statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:27Z","serveDeploymentStatuses":{"FruitMarket":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"MangoStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"OrangeStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"PearStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:27Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"Multiplier":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:27.455Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","isReady":false,"isActive":false} | |
| {"level":"info","ts":"2024-04-16T23:11:27.566Z","logger":"controllers.RayService","msg":"Mark cluster as waiting for Serve deployments","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f","rayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"default","name":"rayservice-sample-raycluster-gt2bm"}} | |
| {"level":"info","ts":"2024-04-16T23:11:27.566Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62c19044-5ee1-493e-b2a6-e8276a0a3c3f"} | |
| {"level":"info","ts":"2024-04-16T23:11:29.568Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:29.568Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:29.568Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:29.568Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:29.642Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","prev statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:27Z","serveDeploymentStatuses":{"FruitMarket":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"MangoStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"OrangeStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"PearStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:27Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"Multiplier":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:27Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"DEPLOYING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"UPDATING"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"UPDATING"},"PearStand":{"name":"PearStand","status":"UPDATING"}}},"math_app":{"name":"math_app","status":"DEPLOYING","deployments":{"Adder":{"name":"Adder","status":"UPDATING"},"Multiplier":{"name":"Multiplier","status":"UPDATING"},"Router":{"name":"Router","status":"UPDATING"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:29.642Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","new statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:29Z","serveDeploymentStatuses":{"FruitMarket":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"OrangeStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"PearStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:29Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"Multiplier":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:29.642Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","isReady":false,"isActive":false} | |
| {"level":"info","ts":"2024-04-16T23:11:29.776Z","logger":"controllers.RayService","msg":"Mark cluster as waiting for Serve deployments","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476","rayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"default","name":"rayservice-sample-raycluster-gt2bm"}} | |
| {"level":"info","ts":"2024-04-16T23:11:29.776Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ade2672a-7cca-4f68-8793-9390cb21d476"} | |
| {"level":"info","ts":"2024-04-16T23:11:31.777Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:31.777Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:31.777Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:31.777Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:31.785Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","prev statuses":{"fruit_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:29Z","serveDeploymentStatuses":{"FruitMarket":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"OrangeStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"PearStand":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:29Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"Multiplier":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:29Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"DEPLOYING","deployments":{"Adder":{"name":"Adder","status":"UPDATING"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"UPDATING"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:31.785Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:31Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:31Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:31Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:31.785Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","isReady":false,"isActive":false} | |
| {"level":"info","ts":"2024-04-16T23:11:31.937Z","logger":"controllers.RayService","msg":"Mark cluster as waiting for Serve deployments","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2","rayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"default","name":"rayservice-sample-raycluster-gt2bm"}} | |
| {"level":"info","ts":"2024-04-16T23:11:31.937Z","logger":"controllers.RayService","msg":"Ray Serve applications are not ready to serve requests: checking again in 2ss","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cc64572-62ce-40f0-8b8e-1bb0f9dff3e2"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.938Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.938Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.938Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.938Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:31Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"}}},"math_app":{"status":"DEPLOYING","healthLastUpdateTime":"2024-04-16T23:11:31Z","serveDeploymentStatuses":{"Adder":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:31Z"},"Router":{"status":"UPDATING","healthLastUpdateTime":"2024-04-16T23:11:31Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","isReady":true,"isActive":false} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","ActiveRayClusterName":"","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the pending Ray cluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:33.947Z","logger":"controllers.RayService","msg":"Create a Kubernetes Service for RayService serviceType headService","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.323Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.323Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:34.323Z","logger":"controllers.RayService","msg":"Create a Kubernetes Service for RayService serviceType serveService","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Running","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Running, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:11:34.324Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.324Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"dbe6414e-de83-453e-be98-ed3006e1e02d","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| E0416 23:11:34.494609 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:11:34.593Z","logger":"controllers.RayService","msg":"inconsistentRayServiceStatus RayService ServiceStatus changed from WaitForServeDeploymentReady to Running","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa1251aa-44ae-4d68-b28d-9996625adf2c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.689Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:34Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:34Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:34Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:34.698Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.699Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:34.699Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:34.699Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"745b0602-86e5-417c-a94c-007aab74d08c"} | |
| E0416 23:11:35.947284 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.690Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:33Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:33Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:36.698Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.699Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.699Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:36.699Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| {"level":"info","ts":"2024-04-16T23:11:36.699Z","logger":"controllers.RayService","msg":"inconsistentRayServiceStatus RayService NumServeEndpoints changed from 0 to 1","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c873b2c3-d72b-47b9-984c-35c123998dd5"} | |
| E0416 23:11:38.653729 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.851Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:38Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:38Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:38Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:38.859Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.860Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:38.860Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:38.860Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"734f52f1-65a3-4a5e-bb72-22c79f9a7218"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.861Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:40Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:40Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:40Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:40.869Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.870Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:40.870Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:40.870Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"25ea966c-031e-4571-878b-4c8cf28ecbb2"} | |
| E0416 23:11:42.119228 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.871Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:42Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:42Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:42Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:42.879Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.880Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:42.880Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:42.880Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6b7f5a44-4992-4754-b9a0-2d3f12bca992"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.881Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:44Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:44Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:44Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:44.889Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.890Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:44.890Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:44.890Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c1a3169c-23f2-4b35-974a-70565e6937ec"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.891Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:46Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:46Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:46Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:46.902Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d678efe2-d255-41fb-afd9-6acdc10184b1"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.903Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:48Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:48Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:48Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:48.911Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.912Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:48.912Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:48.912Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f5ed81f-d00a-48e9-805c-04f93ccdac02"} | |
| E0416 23:11:50.826274 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.913Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:50Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:50Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:50Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:50.923Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.924Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:50.924Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:50.924Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bcf12f0b-bfb3-414b-ab62-b610101c1199"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.925Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:52Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:52Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:52Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:52.933Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.934Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:52.934Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:52.934Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8d00743-653a-47d9-8bc1-424169f66e07"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.935Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:54Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:54Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:54Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:54.943Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.944Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:54.944Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:54.944Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"208f070e-a0dc-4461-8456-a5ecfade8d1a"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.944Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.944Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.944Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.945Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.945Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.945Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:56.953Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:56.953Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:56Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:56Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:56Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:56.953Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:56.953Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:57.034Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc"} | |
| {"level":"info","ts":"2024-04-16T23:11:57.034Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:57.034Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:57.034Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc"} | |
| {"level":"info","ts":"2024-04-16T23:11:57.035Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:57.035Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:57.035Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bc71d4cb-cee4-4bfe-ab1a-97054fb364fc"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.035Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:59Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:59Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:59Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:59.043Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.044Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:11:59.044Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:11:59.044Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc32292f-7b4b-4c1e-846b-41156eabb519"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.045Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:01Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:01Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:01Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:01.053Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:01.054Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"4b645bfc-5f32-440e-8121-b369bc678367"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.055Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:03Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:03Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:03Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.063Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:03.064Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:03.064Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74f91911-5b4c-4137-9aa2-22bdb3ba1ecb"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.065Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:05Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:05Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:05Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:05.074Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.075Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:05.075Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:05.075Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b0302df2-403e-4a0f-b646-f4099f9ad182"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.076Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:07Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:07Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:07Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:07.084Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.085Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:07.085Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:07.085Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"10ed0ea8-df25-40db-948e-05e2557d73a7"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.086Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:09Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:09Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:09Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.094Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:09.095Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.095Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:09.095Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:09.095Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c9a664c8-a030-4477-ad02-d08d77488168"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.096Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:11Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:11Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:11Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:11.105Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.106Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:11.106Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:11.106Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3b09a229-ad7c-4103-b1d5-16cf973ab6be"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.107Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:13Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:13Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:13Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:13.115Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.116Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:13.116Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:13.116Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e8e6ba8c-8c64-4bab-9904-9b0c1193c13c"} | |
| E0416 23:12:14.205874 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.117Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:15Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:15Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:15Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.126Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.127Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:15.127Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.134Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:15.134Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:15.134Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"eaf22662-2c7e-4124-a332-f3656d916045"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.134Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.134Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.134Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.135Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.135Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.135Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:17Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:17Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:17Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:17.143Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.144Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:17.144Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:17.144Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6e4e57c0-7a13-44fc-b5d6-99c6be71af66"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.145Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:19Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:19Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:19Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:19.153Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.154Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:19.154Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:19.154Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"74c78f5a-1d26-44bf-b194-edc919d5e5aa"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.154Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.154Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.154Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.155Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.155Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.155Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:21Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:21Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:21Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:21.163Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.164Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:21.164Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:21.164Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d008828b-6d5e-4ebe-8e97-e77da2dd2d70"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.165Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:23Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:23Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:23Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:23.173Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.174Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:23.174Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:23.174Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c7208ea9-a066-4a3d-8034-9b4167da242b"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.175Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:25Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:25Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:25Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:25.183Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.184Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:25.184Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:25.184Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"45e1bf11-37f4-486d-b936-3cd541cd9c3c"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.184Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.184Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.184Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.185Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.185Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.185Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:27Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:27Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:27Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:27.193Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ef560132-3b57-41f7-b022-c0c7a538e1f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.194Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:29Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:29Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:29Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:29.204Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.205Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:29.205Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:29.205Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"cb88feff-fb25-45b8-ac5a-f4b7c05edc03"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.206Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:31Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:31Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:31Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:31.215Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.234Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:31.234Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:31.234Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"15eb2331-b7d9-4173-b0b5-c02923dcd36c"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.235Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.235Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.235Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.235Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.235Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.236Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:33Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:33Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:33Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:33.244Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:33.245Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"3a0a56e9-69ff-4a72-8702-56233b25ffdd"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.246Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.253Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:35.253Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:35Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:35Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:35Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:35.253Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:35.253Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:35.254Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7118ad4d-bdbe-4bbd-a057-4dfed6a0cfa2"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.256Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.263Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:37Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:37Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:37Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:37.264Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ac6c1f12-3110-456a-b55a-dec8cca9d3f3"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.266Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:39Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:39Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:39Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:39.274Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.275Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:39.275Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:39.275Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5c690ae3-53c4-48d1-80ff-cfaa10691aa4"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.276Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:41Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:41Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:41Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:41.284Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.285Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:41.285Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:41.285Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"894ab6b0-3671-4ed0-9056-762c9ede4274"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.286Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:43Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:43Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:43Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:43.294Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"f8f18ea6-77c0-49a2-a73d-8964f841d5f6"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.295Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.302Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:45.302Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:45Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:45Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:45Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:45.303Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e9e5e51e-ae92-404e-b775-25e1811e2f7c"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.304Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:47Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:47Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:47Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:47.312Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.313Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:47.313Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:47.313Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"740ea0eb-27ac-40bd-b7f7-99764e5368df"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.314Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:49Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:49Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:49Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:49.322Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c10f4818-4e41-4764-9649-77765f6ca58c"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.323Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:51Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:51Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:51Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:51.331Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"81512399-8e9c-4190-a7f4-3572cb84cf8f"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.332Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:53Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:53Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:53Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:53.340Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7efc169d-8bfc-4ccb-9014-212d654eeb50"} | |
| E0416 23:12:54.469474 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.341Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:55Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:55Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:55Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:55.349Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.350Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:55.350Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:55.350Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"86b64cb1-87f3-4d05-bd2d-538e2e836452"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.351Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:57Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:57Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:57Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:57.359Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.360Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:57.360Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:57.360Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d89f4137-12fd-49f9-97f0-a00120cd29f0"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.361Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:59Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:12:59Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:12:59Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:59.369Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.370Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:12:59.370Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:12:59.370Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f491bda-4b09-41e6-a4a9-5cbda00b9aef"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.371Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:01Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:01Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:01Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:01.379Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"91d44a1e-cef8-45e1-b370-5d3305e1b25e"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.380Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.387Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:03.387Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:03Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:03Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:03Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:03.387Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:03.387Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.387Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:03.388Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"319678e7-245b-4bd3-8570-a80d1cade6ed"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.389Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.397Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:05Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:05Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:05Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:05.398Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"ed0a292a-e892-44cf-8c65-22bc61167f71"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.399Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:07Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:07Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:07Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.407Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:07.408Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:07.408Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"60afdcd9-dabb-40ae-a2d3-206e42f7ca7d"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.408Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:09Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:09Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:09Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:09.416Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.417Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:09.417Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:09.417Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"8473cdcd-2e7c-4c1a-94b3-8b9eab229c8b"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.418Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:11Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:11Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:11Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:11.426Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.427Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:11.427Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:11.427Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c5d91f1a-2682-4371-8dc0-fc0d8704db37"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.428Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:13Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:13Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:13Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.436Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:13.437Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:13.437Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"da3dfa62-dcbe-4782-b802-611b5c450127"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.438Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:15Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:15Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:15Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:15.445Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.446Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:15.446Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:15.446Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"bee090d8-7c41-441c-abd2-61b616b336c6"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.447Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:17Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:17Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:17Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:17.455Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fc72c80f-8bb5-42fe-83dc-a027c5252247"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.456Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:19Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:19Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:19Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.464Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:19.465Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:19.465Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d6c5677e-be5a-4e7f-8618-fb809c2a9ff5"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.467Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:21Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:21Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:21Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:21.475Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.534Z","logger":"controllers.RayService","msg":"CheckHealth fails","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.541Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.541Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:21.541Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"082f975d-d100-4bc0-bba2-631f0f73fb05"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Running","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Running, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:21.542Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"eb7761c3-588e-490a-9c7b-30adbeb588ee","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.542Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:23Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:23Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:23Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:23.550Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.557Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.557Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:23.557Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"e18ac6de-5e9f-49ee-ac19-f2900a1ff82b"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0","1 head service found":"rayservice-sample-raycluster-gt2bm-head-svc"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0","Found 1 head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","Pod status":"Running","Pod restart policy":"Always","Ray container terminated status":"nil"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0","head Pod":"rayservice-sample-raycluster-gt2bm-head-9724b","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod rayservice-sample-raycluster-gt2bm-head-9724b. The Pod status is Running, and the Ray container terminated status is nil."} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"Environment variable RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV is not set, using default value of 300 seconds","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0","cluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:23.558Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"rayservice-sample-raycluster-gt2bm","namespace":"default"},"reconcileID":"f4df4854-ee0b-4db9-9088-be9c32a968f0","cluster name":"rayservice-sample-raycluster-gt2bm","seconds":300} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.558Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:25Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:25Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:25Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:25.566Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.567Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:25.567Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:25.567Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"6dd3d79a-4a14-4cb2-95b0-5a6ef2867d4a"} | |
| E0416 23:13:26.023043 1 reflector.go:147] pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:229: Failed to watch *v1.Endpoints: unknown (get endpoints) | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.568Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:27Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:27Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:27Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.576Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:27.577Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:27.577Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"df1e783d-187d-4469-82fd-b3f10b8e003b"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.577Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.578Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.578Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.578Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.578Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.578Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:29Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:29Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:29Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:29.585Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.586Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:29.586Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:29.586Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7ed0556f-8752-47a9-ab0e-bcbba897ce59"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.586Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.587Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.587Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.587Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.587Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.587Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:31Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:31Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:31Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:31.595Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:31.596Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"16d8b3c9-a65f-4226-9f20-903ea5b4091e"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.596Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:33Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:33Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:33Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:33.604Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.605Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:33.605Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:33.605Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"111500f2-c3ba-4388-b73e-e3fcf14a58ab"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.606Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.606Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.606Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.607Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.607Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.607Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.614Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:35.614Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:35Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:35Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:35Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:35.614Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:35.614Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.614Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:35.615Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"02a7421b-79d3-443e-98be-f3379398ec85"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.616Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:37Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:37Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:37Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:37.624Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"fa16819c-5d9d-46a8-9b53-4e536a9bdf14"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.626Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:39Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:39Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:39Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:39.634Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.635Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:39.635Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:39.635Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"918c66cc-22b3-4117-99f7-6bfc86e69b20"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.635Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:41Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:41Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:41Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:41.643Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.644Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:41.644Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:41.644Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"611f97de-72cf-467a-9594-ea63b33adc28"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.644Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.644Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.644Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.644Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.645Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.645Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.652Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:43.652Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:43Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:43Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:43Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:43.653Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"baeca809-fcff-4a5d-be6e-96fff9fc790b"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.654Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:45Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:45Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:45Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:45.662Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.663Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:45.663Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:45.663Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"2ca675f7-15c4-4235-8578-d464cdb7e3c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.663Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:47Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:47Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:47Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:47.671Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.672Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:47.672Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:47.672Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"07a950ba-6c30-4d85-b371-72c21948cd4e"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.673Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:49Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:49Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:49Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:49.681Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.682Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:49.682Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:49.682Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b144bc78-6f5d-4fd2-a6b3-f8e8173992f2"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.683Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:51Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:51Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:51Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:51.694Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.695Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:51.695Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:51.695Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"b5d716f4-c847-4925-ad66-83df33f21978"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.695Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.703Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:53.703Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:53Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:53Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:53Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:53.703Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:53.703Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.703Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:53.704Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"be0d632f-4bf3-4686-91bb-bd58f33d11b6"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.705Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:55Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:55Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:55Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:55.713Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.714Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:55.714Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:55.714Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c26fb47d-4828-4748-be82-67ca0bfdbf04"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.715Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:57Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:57Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:57Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:57.723Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1facf614-a06d-4fb8-8223-e29d04f1b9c0"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.724Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:59Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:13:59Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:13:59Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:59.732Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.733Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:13:59.733Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:13:59.733Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"dde64924-bf49-493e-be7c-39764787b410"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.734Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:01Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:01Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:01Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:01.742Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.743Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:01.743Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:01.743Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"5f26178b-71cf-4a95-b2bb-90d67e33952b"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.744Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:03Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:03Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:03Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:03.752Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"72c30937-0992-44e9-b1b6-e998eef51eca"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.753Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.754Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.754Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.754Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.754Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.754Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:05Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:05Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:05Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:05.761Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.762Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:05.762Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:05.762Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"62153f07-3433-462d-be2f-b5f07a7c7b5a"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.763Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:07Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:07Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:07Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:07.771Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"d0db0ac7-b414-404b-8a95-0869f63d5bff"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.772Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:09Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:09Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:09Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.780Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.781Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:09.781Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.781Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:09.781Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:09.781Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"0f33c5d9-fa8c-4cd6-87f5-2bc13f3f589e"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.783Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:11Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:11Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:11Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:11.790Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.791Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:11.791Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:11.791Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"7548df2a-9ad5-4293-89f3-768d9d16c7cc"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.792Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:13Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:13Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:13Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:13.800Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.801Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:13.801Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:13.801Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"1cd5858d-1954-4739-9413-7242720f3eb3"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.802Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n","current Serve config":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: Multiplier\n num_replicas: 1\n user_config:\n factor: 5\n ray_actor_options:\n num_cpus: 0.1\n - name: Router\n num_replicas: 1\n"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","prev statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:11:36Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:11:36Z"}}}},"serve statuses":{"fruit_app":{"name":"fruit_app","status":"RUNNING","deployments":{"FruitMarket":{"name":"FruitMarket","status":"HEALTHY"},"MangoStand":{"name":"MangoStand","status":"HEALTHY"},"OrangeStand":{"name":"OrangeStand","status":"HEALTHY"},"PearStand":{"name":"PearStand","status":"HEALTHY"}}},"math_app":{"name":"math_app","status":"RUNNING","deployments":{"Adder":{"name":"Adder","status":"HEALTHY"},"Multiplier":{"name":"Multiplier","status":"HEALTHY"},"Router":{"name":"Router","status":"HEALTHY"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"getAndCheckServeStatus","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","new statuses":{"fruit_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:15Z","serveDeploymentStatuses":{"FruitMarket":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"},"MangoStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"},"OrangeStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"},"PearStand":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"}}},"math_app":{"status":"RUNNING","healthLastUpdateTime":"2024-04-16T23:14:15Z","serveDeploymentStatuses":{"Adder":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"},"Multiplier":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"},"Router":{"status":"HEALTHY","healthLastUpdateTime":"2024-04-16T23:14:15Z"}}}}} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"Check serve health","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","isReady":true,"isActive":true} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"updateRayClusterInfo","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","ActiveRayClusterName":"rayservice-sample-raycluster-gt2bm","healthyClusterName":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"Reconciling the ingress and service resources on the active Ray cluster. No pending Ray cluster found.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","serviceType":"headService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","newSvc":{"namespace":"default","name":"rayservice-sample-head-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's headService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","serviceType":"serveService","RayService name":"rayservice-sample","RayService namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"reconcileServices","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40","newSvc":{"namespace":"default","name":"rayservice-sample-serve-svc"}} | |
| {"level":"info","ts":"2024-04-16T23:14:15.811Z","logger":"controllers.RayService","msg":"RayCluster rayservice-sample-raycluster-gt2bm's serveService has already exists, skip Update","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"97882720-52b4-4434-877d-81b6b870fd40"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"Active Ray cluster config matches goal config. No need to update RayCluster.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"Reconciling the Serve component. Only the active Ray cluster exists.","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"Check the head Pod status of the pending RayCluster","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc","RayCluster name":"rayservice-sample-raycluster-gt2bm"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc","head service name":"rayservice-sample-raycluster-gt2bm-head-svc","namespace":"default"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"FetchHeadServiceURL","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc","head service URL":"rayservice-sample-raycluster-gt2bm-head-svc.default.svc.cluster.local:8265","port":"dashboard"} | |
| {"level":"info","ts":"2024-04-16T23:14:17.812Z","logger":"controllers.RayService","msg":"shouldUpdate","RayService":{"name":"rayservice-sample","namespace":"default"},"reconcileID":"c3133770-acf7-4224-b000-6e85d6a41bcc","shouldUpdateServe":false,"reason":"Current Serve config matches cached Serve config, and some deployments have been deployed for cluster rayservice-sample-raycluster-gt2bm","cachedServeConfig":"applications:\n - name: fruit_app\n import_path: fruit.deployment_graph\n route_prefix: /fruit\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: MangoStand\n num_replicas: 2\n user_config:\n price: 3\n ray_actor_options:\n num_cpus: 0.1\n - name: OrangeStand\n num_replicas: 1\n user_config:\n price: 2\n ray_actor_options:\n num_cpus: 0.1\n - name: PearStand\n num_replicas: 1\n user_config:\n price: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: FruitMarket\n num_replicas: 1\n ray_actor_options:\n num_cpus: 0.1\n - name: math_app\n import_path: conditional_dag.serve_dag\n route_prefix: /calc\n runtime_env:\n working_dir: \"https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip\"\n deployments:\n - name: Adder\n num_replicas: 1\n user_config:\n increment: 3\n ra |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment