Created
March 24, 2022 17:53
-
-
Save nicktrav/a16a050d6154def761d3a70f031c4507 to your computer and use it in GitHub Desktop.
78408
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| I220324 07:25:46.405798 380 kv/kvserver/pkg/kv/kvserver/replica_command.go:1638 ⋮ [n1,s1,r407/2:‹/Table/106/1/33{784918-835414}›] 2136 could not successfully add and upreplicate LEARNER replica(s) on [n9,s9], rolling back: change replicas of r407 failed: descriptor changed: [expected] r407:‹/Table/106/1/3{3784918-9061000}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=94, sticky=1648110326.133963771,0] != [actual] r407:‹/Table/106/1/33{784918-835414}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=95, sticky=1648110326.133963771,0] | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 ⋮ [n1,s1,store-rebalancer] 2137 unable to relocate range to [n9,s9 n1,s1 n7,s7]: while carrying out changes [{ADD_VOTER n9,s9} {REMOVE_VOTER n4,s4}]: change replicas of r407 failed: descriptor changed: [expected] r407:‹/Table/106/1/3{3784918-9061000}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=94, sticky=1648110326.133963771,0] != [actual] r407:‹/Table/106/1/33{784918-835414}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=95, sticky=1648110326.133963771,0] | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +(1) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | (opaque error wrapper) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | type name: github.com/cockroachdb/errors/withstack/*withstack.withStack | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | reportable 0: | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).relocateReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2851 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminRelocateRange | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2769 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithoutRangeID | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:177 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:100 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1006 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:989 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1058 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:554 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2060 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1608 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1210 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:831 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:222 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:968 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.sendAndFill | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:830 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:853 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).AdminRelocateRange | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:677 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).rebalanceStore.func2 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:370 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).rebalanceStore | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:369 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:225 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | runtime.goexit | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | GOROOT/src/runtime/asm_amd64.s:1581 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (2) while carrying out changes [{ADD_VOTER n9,s9} {REMOVE_VOTER n4,s4}] | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (3) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | (opaque error wrapper) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | type name: github.com/cockroachdb/errors/withstack/*withstack.withStack | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | reportable 0: | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.execChangeReplicasTxn | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2311 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).initializeRaftLearners | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:1677 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).changeReplicasImpl | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:1068 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).ChangeReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:996 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:933 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithoutRangeID | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:177 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:100 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1006 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:989 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1058 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:554 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2060 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1608 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1210 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:831 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:222 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:968 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.sendAndFill | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:830 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:853 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).AdminChangeReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:651 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).relocateReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2849 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminRelocateRange | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2769 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithoutRangeID | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:177 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:100 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (4) change replicas of r407 failed | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (5) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | (opaque error wrapper) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | type name: github.com/cockroachdb/cockroach/pkg/kv/kvserver/*kvserver.benignError | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (6) secondary error attachment | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 ⋮ [n1,s1,store-rebalancer] 2137 + | ‹unexpected value: raw_bytes:"~o1\367\003\010\227\003\022\007\362\211\371\002\003\204V\032\007\362\211\371\002\004I\226\"\006\010\004\020\004\030\001\"\006\010\001\020\001\030\002\"\006\010\007\020\007\030\003(\0040_:\n\010\373\367\370\336\300\363\320\357\026" timestamp:<wall_time:1648106743110430524 > › | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 ⋮ [n1,s1,store-rebalancer] 2137 + | (1) ‹unexpected value: raw_bytes:"~o1\367\003\010\227\003\022\007\362\211\371\002\003\204V\032\007\362\211\371\002\004I\226\"\006\010\004\020\004\030\001\"\006\010\001\020\001\030\002\"\006\010\007\020\007\030\003(\0040_:\n\010\373\367\370\336\300\363\320\357\026" timestamp:<wall_time:1648106743110430524 > › | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | Error types: (1) *roachpb.ConditionFailedError | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (7) forced error mark | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 ⋮ [n1,s1,store-rebalancer] 2137 + | ‹"should retry with updated descriptor"› | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/errors/withstack/*withstack.withStack:: | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Wraps: (8) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | (opaque error wrapper) | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | type name: github.com/cockroachdb/errors/withstack/*withstack.withStack | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | reportable 0: | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.newDescChangedError | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/markers.go:51 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.execChangeReplicasTxn | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2308 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).initializeRaftLearners | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:1677 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).changeReplicasImpl | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:1068 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).ChangeReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:996 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:933 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithoutRangeID | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:177 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:100 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1006 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:989 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/server/node.go:1058 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:554 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2060 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1608 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1210 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:831 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:222 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:968 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.sendAndFill | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:830 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:853 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv.(*DB).AdminChangeReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/db.go:651 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).relocateReplicas | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2849 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminRelocateRange | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_command.go:2769 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:951 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithoutRangeID | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:177 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 + | github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:100 | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 ⋮ [n1,s1,store-rebalancer] 2137 +Wraps: (9) descriptor changed: [expected] r407:‹/Table/106/1/3{3784918-9061000}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=94, sticky=1648110326.133963771,0] != [actual] r407:‹/Table/106/1/33{784918-835414}› [(n4,s4):1, (n1,s1):2, (n7,s7):3, next=4, gen=95, sticky=1648110326.133963771,0] | |
| E220324 07:25:46.413081 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:378 â‹® [n1,s1,store-rebalancer] 2137 +Error types: (1) *errbase.opaqueWrapper (2) *errutil.withPrefix (3) *errbase.opaqueWrapper (4) *errutil.withPrefix (5) *errbase.opaqueWrapper (6) *secondary.withSecondaryError (7) *markers.withMark (8) *errbase.opaqueWrapper (9) *errutil.leafError | |
| I220324 07:25:46.414426 380 kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:349 â‹® [n1,s1,store-rebalancer] 2138 ran out of replicas worth transferring and qps (753.89) is still above desired threshold (626.90); will check again soon |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment