https://www.gooddata.com/ logo
Join Slack
Powered by
Gooddata-cn upgrade 2.1.0 error We are are curren...
# gooddata-cn
k
Gooddata-cn upgrade 2.1.0 error We are are currently upgrading(helm upgrade) gooddata-cn from 1.7.2 to 2.1.0 and pulsar from 2.7.4 to 2.9.2. During upgrade 4 services are not starting up. gooddata-cn-auth-service, gooddata-cn-calcique, gooddata-cn-metadata-api, gooddata-cn-result-cache. All the failed services are throwing up the same error. "Topic level policies is disabled, to enable the topic level policy and retry.HTTP 405 Method Not Allowed". Here the is the stacktrace from one pod. WARNING: All illegal access operations will be denied in a future release {"ts":"2022-09-13 203821.574","level":"WARN","logger":"org.apache.pulsar.client.admin.internal.BaseResource","thread":"AsyncHttpClient-11-1","msg":"[http://pulsar-broker.pulsar:8080/admin/v2/persistent/gooddata-cn/gooddata-cn/sql.select] Failed to perform http put request: org.apache.pulsar.shade.javax.ws.rs.ClientErrorException: HTTP 409 Conflict"} {"ts":"2022-09-13 203821.690","level":"ERROR","logger":"org.springframework.boot.SpringApplication","thread":"main","msg":"Application run failed","exc":"org.apache.pulsar.client.admin.PulsarAdminException$NotAllowedException: Topic level policies is disabled, to enable the topic level policy and retry.\n\tat org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:236)\n\tat org.apache.pulsar.client.admin.internal.TopicPoliciesImpl.setMessageTTL(TopicPoliciesImpl.java:499)\n\tat com.gooddata.tiger.pulsar.PulsarAutoConfiguration.producerBeanFactory$lambda-1(PulsarAutoConfiguration.kt:132)\n\tat org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:325)\n\tat org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:198)\n\tat org.springframework.context.support.AbstractApplicationContext.invokeBeanFactoryPostProcessors(AbstractApplicationContext.java:746)\n\tat org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:564)\n\tat org.springframework.boot.web.reactive.context.ReactiveWebServerApplicationContext.refresh(ReactiveWebServerApplicationContext.java:64)\n\tat org.springframework.boot.SpringApplication.refresh(SpringApplication.java:745)\n\tat org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:420)\n\tat org.springframework.boot.SpringApplication.run(SpringApplication.java:307)\n\tat org.springframework.boot.SpringApplication.run(SpringApplication.java:1317)\n\tat org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)\n\tat com.gooddata.tiger.calcique.CalciqueApplicationKt.main(CalciqueApplication.kt:59)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)\n\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)\n\tat java.base/java.lang.reflect.Method.invoke(Unknown Source)\n\tat org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)\n\tat org.springframework.boot.loader.Launcher.launch(Launcher.java:108)\n\tat org.springframework.boot.loader.Launcher.launch(Launcher.java:58)\n\tat org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:88)\nCaused by: org.apache.pulsar.shade.javax.ws.rs.NotAllowedException: HTTP 405 Method Not Allowed\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.convertToException(JerseyInvocation.java:951)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.translate(JerseyInvocation.java:755)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.lambda$invoke$1(JerseyInvocation.java:675)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.call(JerseyInvocation.java:697)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.lambda$runInScope$3(JerseyInvocation.java:691)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:292)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:274)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:205)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:390)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.runInScope(JerseyInvocation.java:691)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.invoke(JerseyInvocation.java:674)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation$Builder.method(JerseyInvocation.java:450)\n\tat org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation$Builder.post(JerseyInvocation.java:351)\n\tat org.apache.pulsar.client.admin.internal.TopicPoliciesImpl.setMessageTTL(TopicPoliciesImpl.java:497)\n\t... 20 more\n"} {"ts":"2022-09-13 203821.707","level":"INFO","logger":"org.apache.pulsar.client.impl.PulsarClientImpl","thread":"main","msg":"Client closing. URL: pulsar://pulsar-broker.pulsar:6650"} Exception in thread "main" java.lang.reflect.InvocationTargetException at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49) at org.springframework.boot.loader.Launcher.launch(Launcher.java:108) at org.springframework.boot.loader.Launcher.launch(Launcher.java:58) at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:88) Caused by: java.lang.reflect.UndeclaredThrowableException at org.springframework.util.ReflectionUtils.rethrowRuntimeException(ReflectionUtils.java:147) at org.springframework.boot.SpringApplication.handleRunFailure(SpringApplication.java:809) at org.springframework.boot.SpringApplication.run(SpringApplication.java:317) at org.springframework.boot.SpringApplication.run(SpringApplication.java:1317) at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306) at com.gooddata.tiger.calcique.CalciqueApplicationKt.main(CalciqueApplication.kt:59) ... 8 more Caused by: org.apache.pulsar.client.admin.PulsarAdminException$NotAllowedException: Topic level policies is disabled, to enable the topic level policy and retry. at org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:236) at org.apache.pulsar.client.admin.internal.TopicPoliciesImpl.setMessageTTL(TopicPoliciesImpl.java:499) at com.gooddata.tiger.pulsar.PulsarAutoConfiguration.producerBeanFactory$lambda-1(PulsarAutoConfiguration.kt:132) at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:325) at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:198) at org.springframework.context.support.AbstractApplicationContext.invokeBeanFactoryPostProcessors(AbstractApplicationContext.java:746) at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:564) at org.springframework.boot.web.reactive.context.ReactiveWebServerApplicationContext.refresh(ReactiveWebServerApplicationContext.java:64) at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:745) at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:420) at org.springframework.boot.SpringApplication.run(SpringApplication.java:307) ... 11 more Caused by: org.apache.pulsar.shade.javax.ws.rs.NotAllowedException: HTTP 405 Method Not Allowed at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.convertToException(JerseyInvocation.java:951) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.translate(JerseyInvocation.java:755) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.lambda$invoke$1(JerseyInvocation.java:675) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.call(JerseyInvocation.java:697) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.lambda$runInScope$3(JerseyInvocation.java:691) at org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:292) at org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:274) at org.apache.pulsar.shade.org.glassfish.jersey.internal.Errors.process(Errors.java:205) at org.apache.pulsar.shade.org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:390) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.runInScope(JerseyInvocation.java:691) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation.invoke(JerseyInvocation.java:674) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation$Builder.method(JerseyInvocation.java:450) at org.apache.pulsar.shade.org.glassfish.jersey.client.JerseyInvocation$Builder.post(JerseyInvocation.java:351) at org.apache.pulsar.client.admin.internal.TopicPoliciesImpl.setMessageTTL(TopicPoliciesImpl.java:497) ... 20 more
j
@Robert Moucha please, help.
r
Hi, we're deeply sorry, but important configuration change related to pulsar helm chart didn't get into documentation. We'll update docs and release nodes ASAP. In order to fix your deployment, please update values for Apache Pulsar helm chart:
Copy code
broker:
  configData:
    ... other settings, like subscriptionExpirationTimeMinutes etc...
    systemTopicEnabled: "true"
    topicLevelPoliciesEnabled: "true"
and redeploy Pulsar with updated values file. Then, GoodData.CN components will reconnect to pulsar automatically.
v
thanks @Robert Moucha
k
@Robert Moucha I redeployed Pulsar with these changes, I can see the stateful set broker pods has these properties. Then I restated the deployments for the failed services in GoodData-cn. But I still the same issue persists.
r
I see, please restart pulsar brokers manually. They don't respect config change by default
alternatively, you may set 
restartPodsOnConfigMapChange: true
to broker's config section in customized-values-pulsar.yaml as hinted in (already updated) configuration https://www.gooddata.com/developers/cloud-native/doc/2.1/deploy-and-install/cloud-native/helm-chart-installation/#use-customized-[…]syaml-for-pulsar
k
@Robert Moucha it worked. Thanks for the quick response. Saved a day for us as we are in North America.
@Robert Moucha here is the latest issue after upgrade. We are getting HTTP 405 for any client for any GoodData-cn org url. Here is the response from a Curl call.
Here is the response from web client. 405 for option request call.
I think these issues are happening because of new api versioning requirement. If you anticipate any other root cause please let us know.
p
we're currently using the following sdk-ui dependencies "@gooddata/sdk-ui": "^8.8.0", "@gooddata/sdk-ui-all": "^8.9.0", "@gooddata/sdk-ui-charts": "^8.8.0", "@gooddata/sdk-ui-dashboard": "^8.8.0", "@gooddata/sdk-ui-ext": "^8.8.0", "@gooddata/sdk-ui-filters": "^8.8.0", "@gooddata/sdk-ui-geo": "^8.8.0", "@gooddata/sdk-ui-pivot": "^8.8.0" what versions of these dependencies do we need to use with GoodData.CN version 2.1.0 ?
k
@Robert Moucha these ui packages above don't generate api endpoints with versions(v1). So our ui clients are currently broken.
r
you need 8.10.0 and newer
đź‘Ť 1
p
Thanks @Robert Moucha
p
@Robert Moucha Do we also need to update sdk-backend-tiger dependency to 8.10.0 to be able to work with GoodData.CN 2.1.0 ?
r
I belive all @gooddata/sdk-ui-* libs should have the same version. @Dan Homola please can you confirm?
d
Yes, exactly, all of the pacakges from the GoodData.UI need to be kept at the same version, otherwise strange errors can happen especially in the React components.
n
Hi @Robert Moucha, there is one issue we are facing. Once auth0 session expires ( after 1 hour), gooddata urls fails for the first time rather then doing reauthentication. If user refresh again, then it works. In network logs, we see the url: /api/v1/entities/workspaces?include=workspaces&page=0&size=250&sort=name returning 500 error (rather than 401). And in k8, there is an error in pod gooddata-cn-metadata-api : 2022-09-19 133934.064 IST{exc: com.gooddata.oauth2.server.common.CookieDecodeException: JWT from cookie decoding error at com.gooddata.oauth2.server.servlet.CookieSecurityContextRepository.loadContext(CookieSecurityContextRepository.kt:75) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilt… +cc @Kshirod Mohanty, @rohit mathur, @Alex Kearns, @Vivek M
Here is the full stack trace: {"exc":"com.gooddata.oauth2.server.common.CookieDecodeException: JWT from cookie decoding error at com.gooddata.oauth2.server.servlet.CookieSecurityContextRepository.loadContext(CookieSecurityContextRepository.kt:75) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:98) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:336) at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:55) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:336) at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:211) at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:183) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:354) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:267) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.cloud.sleuth.instrument.web.servlet.TracingFilter.doFilter(TracingFilter.java:68) at org.springframework.cloud.sleuth.autoconfig.instrument.web.TraceWebServletConfiguration$LazyTracingFilter.doFilter(TraceWebServletConfiguration.java:131) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.zalando.logbook.servlet.LogbookFilter.doFilter(LogbookFilter.java:74) at org.zalando.logbook.servlet.SecureLogbookFilter.doFilter(SecureLogbookFilter.java:32) at org.zalando.logbook.servlet.HttpFilter.doFilter(HttpFilter.java:31) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.springframework.web.filter.ForwardedHeaderFilter.doFilterInternal(ForwardedHeaderFilter.java:156) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78) at org.springframework.cloud.sleuth.instrument.web.tomcat.TraceValve.invoke(TraceValve.java:103) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:360) at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399) at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:890) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1787) at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191) at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.base/java.lang.Thread.run(Unknown Source) Caused by: org.springframework.security.oauth2.jwt.BadJwtException: An error occurred while attempting to decode the Jwt: Expired JWT at org.springframework.security.oauth2.jwt.NimbusJwtDecoder.createJwt(NimbusJwtDecoder.java:180) at org.springframework.security.oauth2.jwt.NimbusJwtDecoder.decode(NimbusJwtDecoder.java:137) at com.gooddata.oauth2.server.servlet.CookieSecurityContextRepository.loadContext(CookieSecurityContextRepository.kt:70) ... 58 more Caused by: com.nimbusds.jwt.proc.BadJWTException: Expired JWT at com.nimbusds.jwt.proc.DefaultJWTClaimsVerifier.verify(DefaultJWTClaimsVerifier.java:342) at com.nimbusds.jwt.proc.DefaultJWTProcessor.verifyClaims(DefaultJWTProcessor.java:271) at com.nimbusds.jwt.proc.DefaultJWTProcessor.process(DefaultJWTProcessor.java:373) at com.nimbusds.jwt.proc.DefaultJWTProcessor.process(DefaultJWTProcessor.java:303) at org.springframework.security.oauth2.jwt.NimbusJwtDecoder.createJwt(NimbusJwtDecoder.java:154) ... 60 more ", "level":"ERROR", "logger":"org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]", "msg":"Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception", "spanId":"43d84843cd777927", "thread":"http-nio-9007-exec-10", "traceId":"43d84843cd777927", "ts":"2022-09-19 065707.905"}
Open in Slack
PreviousNext
Powered by