How to load udf jars in flink program

[刘建刚]

      We are using per-job to load udf jar when start job. Our jar file is
in another path but not flink's lib path. In the main function, we use
classLoader to load the jar file by the jar path. But it reports the
following error when job starts running.
      If the jar file is in lib, everything is ok. But our udf jar file is
managed in a special path. How can I load udf jars in flink program with
only giving the jar path?

org.apache.flink.api.common.InvalidProgramException: Table program
cannot be compiled. This is a bug. Please file an issue.
        at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
        at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
        at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
        at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
        at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
        at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:424)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:290)
        at org.apache.flink.runtime.taskmanager.Task.run(Task.java:723)
        at java.lang.Thread.run(Thread.java:745)
Caused by: org.codehaus.commons.compiler.CompileException: Line 5,
Column 1: Cannot determine simple type name "com"
        at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11877)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6758)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6519)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
        at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
        at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6498)
        at org.codehaus.janino.UnitCompiler.access$14000(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$22$1.visitReferenceType(UnitCompiler.java:6405)
        at org.codehaus.janino.UnitCompiler$22$1.visitReferenceType(UnitCompiler.java:6400)
        at org.codehaus.janino.Java$ReferenceType.accept(Java.java:3983)
        at org.codehaus.janino.UnitCompiler$22.visitType(UnitCompiler.java:6400)
        at org.codehaus.janino.UnitCompiler$22.visitType(UnitCompiler.java:6393)
        at org.codehaus.janino.Java$ReferenceType.accept(Java.java:3982)
        at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
        at org.codehaus.janino.UnitCompiler.access$1300(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$25.getType(UnitCompiler.java:8206)
        at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6798)
        at org.codehaus.janino.UnitCompiler.access$14500(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$22$2$1.visitFieldAccess(UnitCompiler.java:6423)
        at org.codehaus.janino.UnitCompiler$22$2$1.visitFieldAccess(UnitCompiler.java:6418)
        at org.codehaus.janino.Java$FieldAccess.accept(Java.java:4365)
        at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6418)
        at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6414)
        at org.codehaus.janino.Java$Lvalue.accept(Java.java:4203)
        at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6414)
        at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6393)
        at org.codehaus.janino.Java$Rvalue.accept(Java.java:4171)
        at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
        at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6780)
        at org.codehaus.janino.UnitCompiler.access$14300(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6421)
        at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6418)
        at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4279)
        at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6418)
        at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6414)
        at org.codehaus.janino.Java$Lvalue.accept(Java.java:4203)
        at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6414)
        at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6393)
        at org.codehaus.janino.Java$Rvalue.accept(Java.java:4171)
        at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
        at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8960)
        at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4988)
        at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4452)
        at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4425)
        at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5128)
        at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4425)
        at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5589)
        at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3810)
        at org.codehaus.janino.UnitCompiler.access$5900(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3788)
        at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3760)
        at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5128)
        at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3760)
        at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2363)
        at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1497)
        at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1490)
        at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2929)
        at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1490)
        at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1570)
        at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3398)
        at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1360)
        at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1333)
        at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825)
        at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:435)
        at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:218)
        at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:414)
        at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:409)
        at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1417)
        at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:409)
        at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:381)
        at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
        at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:466)
        at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:216)
        at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:207)
        at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:80)
        at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:75)
        at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:33)
        ... 9 more

[Zhu Zhu]

Hi Jiangang,

Does "flink run -j jarpath ..." work for you?
If that jar id deployed to the same path on each worker machine, you can
try "flink run -C classpath ..." as well.

Thanks,
Zhu Zhu

刘建刚 <[hidden email]> 于2019年8月15日周四 下午5:31写道：

>       We are using per-job to load udf jar when start job. Our jar file is
> in another path but not flink's lib path. In the main function, we use
> classLoader to load the jar file by the jar path. But it reports the
> following error when job starts running.
>       If the jar file is in lib, everything is ok. But our udf jar file is
> managed in a special path. How can I load udf jars in flink program with
> only giving the jar path?
>
> org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
> at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
> at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
> at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
> at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
> at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
> at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
> at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:424)
> at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:290)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:723)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.codehaus.commons.compiler.CompileException: Line 5, Column 1: Cannot determine simple type name "com"
> at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11877)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6758)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6519)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
> at org.codehaus.janino.UnitCompiler.getReferenceType(UnitCompiler.java:6532)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6498)
> at org.codehaus.janino.UnitCompiler.access$14000(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$22$1.visitReferenceType(UnitCompiler.java:6405)
> at org.codehaus.janino.UnitCompiler$22$1.visitReferenceType(UnitCompiler.java:6400)
> at org.codehaus.janino.Java$ReferenceType.accept(Java.java:3983)
> at org.codehaus.janino.UnitCompiler$22.visitType(UnitCompiler.java:6400)
> at org.codehaus.janino.UnitCompiler$22.visitType(UnitCompiler.java:6393)
> at org.codehaus.janino.Java$ReferenceType.accept(Java.java:3982)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
> at org.codehaus.janino.UnitCompiler.access$1300(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$25.getType(UnitCompiler.java:8206)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6798)
> at org.codehaus.janino.UnitCompiler.access$14500(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitFieldAccess(UnitCompiler.java:6423)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitFieldAccess(UnitCompiler.java:6418)
> at org.codehaus.janino.Java$FieldAccess.accept(Java.java:4365)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6418)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6414)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:4203)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6414)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6393)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4171)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6780)
> at org.codehaus.janino.UnitCompiler.access$14300(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6421)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6418)
> at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4279)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6418)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6414)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:4203)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6414)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6393)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4171)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6393)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8960)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4988)
> at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4452)
> at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4425)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5128)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4425)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5589)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3810)
> at org.codehaus.janino.UnitCompiler.access$5900(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3788)
> at org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3760)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5128)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3760)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2363)
> at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1497)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1490)
> at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2929)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1490)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1570)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3398)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1360)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1333)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:435)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:218)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:414)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:409)
> at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1417)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:409)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:381)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
> at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:466)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:216)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:207)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:80)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:75)
> at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:33)
> ... 9 more
>
>