[jira] [Created] (FLINK-22084) RescalingITCase fails with adaptive scheduler

[Shang Yuanchun (Jira)]

Dawid Wysakowicz created FLINK-22084:
----------------------------------------

             Summary: RescalingITCase fails with adaptive scheduler
                 Key: FLINK-22084
                 URL: https://issues.apache.org/jira/browse/FLINK-22084
             Project: Flink
          Issue Type: Bug
          Components: Runtime / Checkpointing, Runtime / Coordination
    Affects Versions: 1.13.0
            Reporter: Dawid Wysakowicz
             Fix For: 1.13.0


https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=15934&view=logs&j=8fd9202e-fd17-5b26-353c-ac1ff76c8f28&t=a0a633b8-47ef-5c5a-2806-3c13b9e48228&l=4472

{code}
2021-03-31T22:16:07.8416407Z [ERROR] testSavepointRescalingOutKeyedStateDerivedMaxParallelism[backend = rocksdb](org.apache.flink.test.checkpointing.RescalingITCase)  Time elapsed: 9.945 s  <<< ERROR!
2021-03-31T22:16:07.8417534Z org.apache.flink.runtime.client.JobExecutionException: Job execution failed.
2021-03-31T22:16:07.8418516Z at org.apache.flink.runtime.jobmaster.JobResult.toJobExecutionResult(JobResult.java:144)
2021-03-31T22:16:07.8419281Z at org.apache.flink.test.util.TestUtils.submitJobAndWaitForResult(TestUtils.java:63)
2021-03-31T22:16:07.8420142Z at org.apache.flink.test.checkpointing.RescalingITCase.testSavepointRescalingKeyedState(RescalingITCase.java:251)
2021-03-31T22:16:07.8421173Z at org.apache.flink.test.checkpointing.RescalingITCase.testSavepointRescalingOutKeyedStateDerivedMaxParallelism(RescalingITCase.java:168)
2021-03-31T22:16:07.8421985Z at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2021-03-31T22:16:07.8422651Z at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2021-03-31T22:16:07.8423649Z at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2021-03-31T22:16:07.8424231Z at java.lang.reflect.Method.invoke(Method.java:498)
2021-03-31T22:16:07.8424657Z at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
2021-03-31T22:16:07.8425147Z at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
2021-03-31T22:16:07.8425609Z at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
2021-03-31T22:16:07.8426183Z at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
2021-03-31T22:16:07.8569060Z at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
2021-03-31T22:16:07.8569781Z at org.apache.flink.util.TestNameProvider$1.evaluate(TestNameProvider.java:45)
2021-03-31T22:16:07.8570451Z at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
2021-03-31T22:16:07.8571040Z at org.junit.rules.RunRules.evaluate(RunRules.java:20)
2021-03-31T22:16:07.8571604Z at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
2021-03-31T22:16:07.8572303Z at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
2021-03-31T22:16:07.8573259Z at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
2021-03-31T22:16:07.8573975Z at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
2021-03-31T22:16:07.8574660Z at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
2021-03-31T22:16:07.8575359Z at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
2021-03-31T22:16:07.8576037Z at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
2021-03-31T22:16:07.8576728Z at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
2021-03-31T22:16:07.8577588Z at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
2021-03-31T22:16:07.8578181Z at org.junit.runners.Suite.runChild(Suite.java:128)
2021-03-31T22:16:07.8578771Z at org.junit.runners.Suite.runChild(Suite.java:27)
2021-03-31T22:16:07.8579402Z at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
2021-03-31T22:16:07.8580061Z at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
2021-03-31T22:16:07.8580774Z at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
2021-03-31T22:16:07.8581480Z at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
2021-03-31T22:16:07.8582148Z at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
2021-03-31T22:16:07.8582896Z at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
2021-03-31T22:16:07.8583762Z at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
2021-03-31T22:16:07.8584427Z at org.junit.rules.RunRules.evaluate(RunRules.java:20)
2021-03-31T22:16:07.8585069Z at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
2021-03-31T22:16:07.8585671Z at org.junit.runners.Suite.runChild(Suite.java:128)
2021-03-31T22:16:07.8586254Z at org.junit.runners.Suite.runChild(Suite.java:27)
2021-03-31T22:16:07.8586875Z at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
2021-03-31T22:16:07.8587643Z at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
2021-03-31T22:16:07.8779731Z at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
2021-03-31T22:16:07.8780398Z at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
2021-03-31T22:16:07.8781024Z at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
2021-03-31T22:16:07.8781702Z at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
2021-03-31T22:16:07.8782346Z at org.apache.maven.surefire.junitcore.JUnitCore.run(JUnitCore.java:55)
2021-03-31T22:16:07.8783166Z at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:137)
2021-03-31T22:16:07.8784006Z at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:107)
2021-03-31T22:16:07.8784796Z at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:83)
2021-03-31T22:16:07.8785556Z at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:75)
2021-03-31T22:16:07.8786346Z at org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:158)
2021-03-31T22:16:07.8787299Z at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
2021-03-31T22:16:07.8788104Z at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
2021-03-31T22:16:07.8815851Z at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
2021-03-31T22:16:07.8816576Z at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
2021-03-31T22:16:07.8819737Z Caused by: java.util.concurrent.CompletionException: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Failed to rollback to checkpoint/savepoint Checkpoint Metadata. Max parallelism mismatch between checkpoint/savepoint state and new program. Cannot map operator 20ba6b65f97481d5570070de90e4e791 with max parallelism 13 to new program with max parallelism 128. This indicates that the program has been changed in a non-compatible way after the checkpoint/savepoint.
2021-03-31T22:16:07.8821554Z at org.apache.flink.runtime.concurrent.FutureUtils.lambda$switchExecutor$23(FutureUtils.java:1362)
2021-03-31T22:16:07.8822349Z at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836)
2021-03-31T22:16:07.8823178Z at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811)
2021-03-31T22:16:07.8823948Z at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)
2021-03-31T22:16:07.8824698Z at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:440)
2021-03-31T22:16:07.8825485Z at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:208)
2021-03-31T22:16:07.8826318Z at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77)
2021-03-31T22:16:07.8827203Z at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158)
2021-03-31T22:16:07.8827925Z at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26)
2021-03-31T22:16:07.8828561Z at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21)
2021-03-31T22:16:07.8829192Z at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123)
2021-03-31T22:16:07.8829860Z at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21)
2021-03-31T22:16:07.8830536Z at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170)
2021-03-31T22:16:07.8831187Z at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
2021-03-31T22:16:07.8831853Z at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
2021-03-31T22:16:07.8832477Z at akka.actor.Actor$class.aroundReceive(Actor.scala:517)
2021-03-31T22:16:07.8833155Z at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225)
2021-03-31T22:16:07.8833784Z at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592)
2021-03-31T22:16:07.8834351Z at akka.actor.ActorCell.invoke(ActorCell.scala:561)
2021-03-31T22:16:07.8835134Z at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
2021-03-31T22:16:07.8835693Z at akka.dispatch.Mailbox.run(Mailbox.scala:225)
2021-03-31T22:16:07.8836208Z at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
2021-03-31T22:16:07.8836805Z at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
2021-03-31T22:16:07.8837571Z at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
2021-03-31T22:16:07.8838263Z at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
2021-03-31T22:16:07.8838962Z at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
2021-03-31T22:16:07.8841344Z Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Failed to rollback to checkpoint/savepoint Checkpoint Metadata. Max parallelism mismatch between checkpoint/savepoint state and new program. Cannot map operator 20ba6b65f97481d5570070de90e4e791 with max parallelism 13 to new program with max parallelism 128. This indicates that the program has been changed in a non-compatible way after the checkpoint/savepoint.
2021-03-31T22:16:07.8843188Z at org.apache.flink.runtime.scheduler.adaptive.BackgroundTask.lambda$new$0(BackgroundTask.java:59)
2021-03-31T22:16:07.8843956Z at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:616)
2021-03-31T22:16:07.8844829Z at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591)
2021-03-31T22:16:07.8845596Z at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)
2021-03-31T22:16:07.8846306Z at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
2021-03-31T22:16:07.8846964Z at java.util.concurrent.FutureTask.run(FutureTask.java:266)
2021-03-31T22:16:07.8847833Z at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
2021-03-31T22:16:07.8848782Z at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
2021-03-31T22:16:07.8849613Z at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
2021-03-31T22:16:07.8850360Z at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2021-03-31T22:16:07.8850983Z at java.lang.Thread.run(Thread.java:748)
2021-03-31T22:16:07.8852976Z Caused by: java.lang.IllegalStateException: Failed to rollback to checkpoint/savepoint Checkpoint Metadata. Max parallelism mismatch between checkpoint/savepoint state and new program. Cannot map operator 20ba6b65f97481d5570070de90e4e791 with max parallelism 13 to new program with max parallelism 128. This indicates that the program has been changed in a non-compatible way after the checkpoint/savepoint.
2021-03-31T22:16:07.8854549Z at org.apache.flink.runtime.checkpoint.Checkpoints.loadAndValidateCheckpoint(Checkpoints.java:181)
2021-03-31T22:16:07.8855424Z at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.restoreSavepoint(CheckpointCoordinator.java:1630)
2021-03-31T22:16:07.8856440Z at org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory.tryRestoreExecutionGraphFromSavepoint(DefaultExecutionGraphFactory.java:163)
2021-03-31T22:16:07.8862100Z at org.apache.flink.runtime.scheduler.DefaultExecutionGraphFactory.createAndRestoreExecutionGraph(DefaultExecutionGraphFactory.java:138)
2021-03-31T22:16:07.8863316Z at org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler.createExecutionGraphAndRestoreState(AdaptiveScheduler.java:971)
2021-03-31T22:16:07.8864391Z at org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler.lambda$createExecutionGraphAndRestoreStateAsync$24(AdaptiveScheduler.java:961)
2021-03-31T22:16:07.8865366Z at org.apache.flink.runtime.scheduler.adaptive.BackgroundTask.lambda$new$0(BackgroundTask.java:57)
2021-03-31T22:16:07.8865913Z ... 10 more
{code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)