MacBook-Air:.docker $ docker login –username=leosunnyPassword:
Login Succeeded
MacBook-Air:.docker $ docker pull leosunny/goldengate4bd:12.3.2.1.0
MacBook-Air:~ $ docker run –name=ogg4bd -d leosunny/goldengate4bd:12.3.2.1.0
MacBook-Air:~ $ docker exec -it –user oracle ogg4bd bash
[oracle@64eeba6b8b09 ogg]$ pwd
/u01/app/ogg
[oracle@64eeba6b8b09 ogg]$ ggsci
Oracle GoldenGate for Big Data
Version 12.3.2.1.0 (Build 018)
Oracle GoldenGate Command Interpreter
Version 12.3.0.1.2 OGGCORE_OGGADP.12.3.0.1.2_PLATFORMS_180430.2321
Linux, x64, 64bit (optimized), Generic on May 1 2018 19:58:51
Operating system character set identified as US-ASCII.
Copyright (C) 1995, 2018, Oracle and/or its affiliates. All rights reserved.
GGSCI (64eeba6b8b09) 1> info all
Program Status Group Lag at Chkpt Time Since Chkpt
MANAGER RUNNING
MacBook-Air:~ $ docker inspect ogg4bd
“NetworkSettings”: {
“Bridge”: “”,
“SandboxID”: “8edd4a386ff81adbfcf15becfcd68d701b61077f456456d1180e39b0bc068a21”,
“HairpinMode”: false,
“LinkLocalIPv6Address”: “”,
“LinkLocalIPv6PrefixLen”: 0,
“Ports”: {
“443/tcp”: null
},
“SandboxKey”: “/var/run/docker/netns/8edd4a386ff8”,
“SecondaryIPAddresses”: null,
“SecondaryIPv6Addresses”: null,
“EndpointID”: “78e9576d6926d3b44b7dd677c2081972a22e29044bf57076ce2646124e9387e3”,
“Gateway”: “172.17.0.1”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“IPAddress”: “172.17.0.2”,
“IPPrefixLen”: 16,
“IPv6Gateway”: “”,
“MacAddress”: “02:42:ac:11:00:02”,
“Networks”: {
“bridge”: {
“IPAMConfig”: null,
“Links”: null,
“Aliases”: null,
“NetworkID”: “f777bf88e20e5e6e91d339208606bdcee1763839857cfd2c6430d025885d2c5b”,
“EndpointID”: “78e9576d6926d3b44b7dd677c2081972a22e29044bf57076ce2646124e9387e3”,
“Gateway”: “172.17.0.1”,
“IPAddress”: “172.17.0.2”,
“IPPrefixLen”: 16,
“IPv6Gateway”: “”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“MacAddress”: “02:42:ac:11:00:02”,
“DriverOpts”: null
}
}
MacBook-Air:~ root# docker run -d –network host –name=ogg4bd -v /hadoop:/u01/app/hadoop leosunny/goldengate4bd:12.3.2.1.0
8caeb22d9586f08601533e3b41c364d98635aad846c38ad13323029b8354741d
docker: Error response from daemon: Mounts denied:
The path /hadoop
is not shared from OS X and is not known to Docker.
You can configure shared paths from Docker -> Preferences… -> File Sharing.
See https://docs.docker.com/docker-for-mac/osxfs/#namespaces for more info.
.
MacBook-Air:hadoop root# docker run -d –network host –name=ogg4bd -v /hadoop:/u02/ogg/hadoop leosunny/goldengate4bd:12.3.2.1.0
MacBook-Air:hadoop root# docker exec -it –user oracle ogg4bd bash
[oracle@linuxkit-025000000001 ogg]$ cd /u02/ogg/
[oracle@linuxkit-025000000001 ogg]$ ls -lrt
total 4
drwxr-xr-x 3 root root 4096 May 27 09:16 hadoop
[oracle@linuxkit-025000000001 ogg]$ cd hadoop/
[oracle@linuxkit-025000000001 hadoop]$ ls -lrt
total 0
drwxr-xr-x 12 oracle oinstall 384 Nov 13 2018 hadoop-2.9.2
[oracle@linuxkit-025000000001 hadoop]$ cd hadoop-2.9.2/
[oracle@linuxkit-025000000001 hadoop-2.9.2]$ ls
LICENSE.txt NOTICE.txt README.txt bin etc include lib libexec sbin share
[oracle@linuxkit-025000000001 hadoop]$ hadoop-2.9.2/bin/hadoop fs -ls /
hadoop-2.9.2/bin/hadoop: line 20: which: command not found
dirname: missing operand
Try ‘dirname –help’ for more information.
hadoop-2.9.2/bin/hadoop: line 27: /u02/ogg/hadoop/../libexec/hadoop-config.sh: No such file or directory
hadoop-2.9.2/bin/hadoop: line 169: exec: : not found
MacBook-Air:hadoop root# docker exec -it –user root ogg4bd bash
bash-4.2# yum install which
bash-4.2# cd /u02/ogg/hadoop/hadoop-2.9.2/
bash-4.2# bin/h
hadoop hadoop.cmd hdfs hdfs.cmd
bash-4.2# bin/hadoop fs -ls /
Found 12 items
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-05-24 23:37 /backup
drwxrwx–x+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-11-16 10:14 /catalog
drwxrwx—+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-09-21 10:02 /clusters
drwxrwx—+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-09-10 23:35 /datalake-poc
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-06-28 18:39 /datalaketest
drwxrwx—+ – 40441d22-f0ec-4dcd-a565-7a9d3bfe4af4 df2923f4-c694-4842-b8c3-6ba472efdb55 0 2019-03-24 02:39 /dump
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-06-24 17:01 /mana
drwxrwx—+ – 37989fa2-592e-47e8-baba-6256bdc668f4 df2923f4-c694-4842-b8c3-6ba472efdb55 0 2019-04-02 09:20 /qubole
drwxrwx—+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-08-28 07:00 /raw
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-06-22 14:29 /test_ogg
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2019-03-27 15:46 /tmp
drwxrwxr–+ – beeca136-b683-4ea5-9892-217cdf48355e df2923f4-c694-4842-b8c3-6ba472efdb55 0 2018-10-01 10:39 /user
[oracle@linuxkit-025000000001 testdir]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -put hello01.txt /datalake-poc/raw/apmt_tos/Mobile
[oracle@linuxkit-025000000001 testdir]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -ls /datalake-poc/raw/apmt_tos/Mobile
Found 3 items
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 1366 2019-05-24 14:42 /datalake-poc/raw/apmt_tos/Mobile/README.txt
-rwxrwx—+ 1 a1b0a4a7-c5a1-4a85-849f-3b7a2ec42212 df2923f4-c694-4842-b8c3-6ba472efdb55 2251 2019-05-24 11:04 /datalake-poc/raw/apmt_tos/Mobile/count_initial_load.txt
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 11 2019-05-27 09:38 /datalake-poc/raw/apmt_tos/Mobile/hello01.txt
[oracle@linuxkit-025000000001 testdir]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -cat /datalake-poc/raw/apmt_tos/Mobile/hello01.txt
helloworld
[oracle@linuxkit-025000000001 ogg]$ cp /u01/app/ogg/AdapterExamples/big-data/hdfs/* /u01/app/ogg/dirprm/
[oracle@linuxkit-025000000001 ogg]$ cd /u01/app/ogg/dirprm/
[oracle@linuxkit-025000000001 dirprm]$ ls -lrt
total 12
-rw-r–r– 1 oracle oinstall 10 May 27 09:28 MGR.prm
-rw-r–r– 1 oracle oinstall 1229 May 27 09:43 hdfs.props
-rw-r–r– 1 oracle oinstall 329 May 27 09:43 rhdfs.prm
[oracle@linuxkit-025000000001 ogg]$ cp /u01/app/ogg/AdapterExamples/trail/tr000000000 /u01/app/ogg/dirdat/
[oracle@linuxkit-025000000001 dirprm]$ ggsci
Oracle GoldenGate Command Interpreter
Version 12.3.0.1.2 OGGCORE_OGGADP.12.3.0.1.2_PLATFORMS_180430.2321
Linux, x64, 64bit (optimized), Generic on May 1 2018 19:58:51
Operating system character set identified as US-ASCII.
Copyright (C) 1995, 2018, Oracle and/or its affiliates. All rights reserved.
GGSCI (linuxkit-025000000001) 1> add replicat rhdfs, exttrail /u01/app/ogg/dirdat/tr
REPLICAT added.
GGSCI (linuxkit-025000000001) 2> info all
Program Status Group Lag at Chkpt Time Since Chkpt
MANAGER RUNNING
REPLICAT STOPPED RHDFS 00:00:00 00:00:02
GGSCI (linuxkit-025000000001) 3> start rhdfs
GGSCI (linuxkit-025000000001) 9> stats rhdfs
Sending STATS request to REPLICAT RHDFS …
Start of Statistics at 2019-05-27 11:58:56.
Replicating from QASOURCE.TCUSTMER to QASOURCE.TCUSTMER:
*** Total statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 1.00
Total deletes 0.00
Total discards 0.00
Total operations 6.00
*** Daily statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 1.00
Total deletes 0.00
Total discards 0.00
Total operations 6.00
*** Hourly statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 1.00
Total deletes 0.00
Total discards 0.00
Total operations 6.00
*** Latest statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 1.00
Total deletes 0.00
Total discards 0.00
Total operations 6.00
Replicating from QASOURCE.TCUSTORD to QASOURCE.TCUSTORD:
*** Total statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 3.00
Total deletes 2.00
Total discards 0.00
Total operations 10.00
*** Daily statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 3.00
Total deletes 2.00
Total discards 0.00
Total operations 10.00
*** Hourly statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 3.00
Total deletes 2.00
Total discards 0.00
Total operations 10.00
*** Latest statistics since 2019-05-27 11:58:28 ***
Total inserts 5.00
Total updates 3.00
Total deletes 2.00
Total discards 0.00
Total operations 10.00
End of Statistics.
[oracle@linuxkit-025000000001 hadoop]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -ls /datalake-poc/raw/apmt_tos/Mobile/
Found 5 items
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 905 2019-05-27 11:55 /datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTMER_RHDFS_2019-05-27_11-58-29.028.txt
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 2305 2019-05-27 11:55 /datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTORD_RHDFS_2019-05-27_11-58-33.215.txt
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 1366 2019-05-24 14:42 /datalake-poc/raw/apmt_tos/Mobile/README.txt
-rwxrwx—+ 1 a1b0a4a7-c5a1-4a85-849f-3b7a2ec42212 df2923f4-c694-4842-b8c3-6ba472efdb55 2251 2019-05-24 11:04 /datalake-poc/raw/apmt_tos/Mobile/count_initial_load.txt
-rw-r—–+ 1 95268324-cad2-4090-bf4f-e4c81d11124d df2923f4-c694-4842-b8c3-6ba472efdb55 11 2019-05-27 09:38 /datalake-poc/raw/apmt_tos/Mobile/hello01.txt
[oracle@linuxkit-025000000001 hadoop]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -cat /datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTMER_RHDFS_2019-05-27_11-58-29.028.txt
IQASOURCE.TCUSTMER2015-11-05 18:45:36.0000002019-05-27T11:58:29.01900000000000000000001956CUST_CODEWILLNAMEBG SOFTWARE CO.CITYSEATTLESTATEWA
IQASOURCE.TCUSTMER2015-11-05 18:45:36.0000002019-05-27T11:58:33.21400000000000000000002126CUST_CODEJANENAMEROCKY FLYER INC.CITYDENVERSTATECO
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T11:58:33.36700000000000000000003286CUST_CODEDAVENAMEDAVE’S PLANES INC.CITYTALLAHASSEESTATEFL
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T11:58:33.36700100000000000000003462CUST_CODEBILLNAMEBILL’S USED CARSCITYDENVERSTATECO
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T11:58:33.36700200000000000000003600CUST_CODEANNNAMEANN’S BOATSCITYSEATTLESTATEWA
UQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T11:58:33.36900000000000000000005100CUST_CODEANNNAMECITYNEW YORKSTATENY
GGSCI (linuxkit-025000000001) 12> view report rhdfs
***********************************************************************
Oracle GoldenGate for Big Data
Version 12.3.2.1.0 (Build 018)
Oracle GoldenGate Delivery
Version 12.3.0.1.2 OGGCORE_OGGADP.12.3.0.1.2_PLATFORMS_180430.2321
Linux, x64, 64bit (optimized), Generic on May 1 2018 20:16:04
Copyright (C) 1995, 2018, Oracle and/or its affiliates. All rights reserved.
Starting at 2019-05-27 11:58:19
***********************************************************************
Operating System Version:
Linux
Version #1 SMP Fri Sep 7 08:20:28 UTC 2018, Release 4.9.125-linuxkit
Node: linuxkit-025000000001
Machine: x86_64
soft limit hard limit
Address Space Size : unlimited unlimited
Heap Size : unlimited unlimited
File Size : unlimited unlimited
CPU Time : unlimited unlimited
Process id: 562
Description:
***********************************************************************
** Running with the following parameters **
***********************************************************************
2019-05-27 11:58:19 INFO OGG-03059 Operating system character set identified as US-ASCII.
2019-05-27 11:58:19 INFO OGG-02695 ANSI SQL parameter syntax is used for parameter parsing.
REPLICAT rhdfs
TARGETDB LIBFILE libggjava.so SET property=dirprm/hdfs.props
2019-05-27 11:58:20 INFO OGG-15052 Using Java class path: ggjava/ggjava.jar:ggjava/resources/lib/optional/log4j-api-2.9.1.jar:ggjava/resources/lib/optional/log4j-core-2.9.1.jar:ggjava/resources/lib/o
ptional/log4j-slf4j-impl-2.9.1.jar.
May 27, 2019 11:58:21 AM org.springframework.beans.factory.xml.XmlBeanDefinitionReader loadBeanDefinitions
INFO: Loading XML bean definitions from class path resource [oracle/goldengate/datasource/DataSource-context.xml]
May 27, 2019 11:58:22 AM org.springframework.beans.factory.xml.XmlBeanDefinitionReader loadBeanDefinitions
INFO: Loading XML bean definitions from class path resource [config/ggue-context.xml]
May 27, 2019 11:58:22 AM org.springframework.beans.factory.support.DefaultListableBeanFactory registerBeanDefinition
INFO: Overriding bean definition for bean ‘dataSourceConfig’ with a different definition: replacing [Generic bean: class [oracle.goldengate.datasource.DataSourceConfig]; scope=singleton; abstract=false; l
azyInit=false; autowireMode=0; dependencyCheck=0; autowireCandidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null; defined in class path re
source [oracle/goldengate/datasource/DataSource-context.xml]] with [Generic bean: class [oracle.goldengate.datasource.DataSourceConfig]; scope=singleton; abstract=false; lazyInit=false; autowireMode=0; de
pendencyCheck=0; autowireCandidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null; defined in class path resource [config/ggue-context.xml]]
May 27, 2019 11:58:22 AM org.springframework.beans.factory.support.DefaultListableBeanFactory registerBeanDefinition
INFO: Overriding bean definition for bean ‘versionInfo’ with a different definition: replacing [Generic bean: class [oracle.goldengate.util.VersionInfo]; scope=singleton; abstract=false; lazyInit=false; a
utowireMode=0; dependencyCheck=0; autowireCandidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null; defined in class path resource [oracle/g
oldengate/datasource/DataSource-context.xml]] with [Generic bean: class [oracle.goldengate.util.VersionInfo]; scope=singleton; abstract=false; lazyInit=false; autowireMode=0; dependencyCheck=0; autowireCa
ndidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null; defined in class path resource [config/ggue-context.xml]]
May 27, 2019 11:58:22 AM org.springframework.context.support.AbstractApplicationContext prepareRefresh
INFO: Refreshing org.springframework.context.support.GenericApplicationContext@78047b92: startup date [Mon May 27 11:58:22 UTC 2019]; root of context hierarchy
May 27, 2019 11:58:22 AM org.springframework.core.io.support.PropertiesLoaderSupport loadProperties
INFO: Loading properties file from URL [jar:file:/u01/app/ogg/ggjava/resources/lib/ggdbutil-12.3.2.1.0.018.jar!/oracle/goldengate/datasource/DataSource.properties]
May 27, 2019 11:58:22 AM org.springframework.core.io.support.PropertiesLoaderSupport loadProperties
INFO: Loading properties file from URL [file:/u01/app/ogg/ggjava/resources/config/ggue.properties]
Oracle GoldenGate for Big Data, 12.3.2.1.0.018
Copyright (c) 2007, 2018. Oracle and/or its affiliates. All rights reserved
Built with Java 1.8.0_161 (class version: 52.0)
REPORTCOUNT EVERY 1 MINUTES, RATE
GROUPTRANSOPS 10000
MAP QASOURCE.*, TARGET QASOURCE.*;
2019-05-27 11:58:28 INFO OGG-01815 Virtual Memory Facilities for: COM
anon alloc: mmap(MAP_ANON) anon free: munmap
file alloc: mmap(MAP_SHARED) file free: munmap
target directories:
/u01/app/ogg/dirtmp.
Database Version:
Database Language and Character Set:
2019-05-27 11:58:28 WARNING OGG-00445 Detected migrated group RHDFS, updating DB checkpoint dir from to /u01/app/ogg.
***********************************************************************
** Run Time Messages **
***********************************************************************
2019-05-27 11:58:28 INFO OGG-02243 Opened trail file /u01/app/ogg/dirdat/tr000000000 at 2019-05-27 11:58:28.943153.
2019-05-27 11:58:28 INFO OGG-03506 The source database character set, as determined from the trail file, is UTF-8.
2019-05-27 11:58:28 INFO OGG-06506 Wildcard MAP resolved (entry QASOURCE.*): MAP “QASOURCE”.”TCUSTMER”, TARGET QASOURCE.”TCUSTMER”.
2019-05-27 11:58:28 INFO OGG-02756 The definition for table QASOURCE.TCUSTMER is obtained from the trail file.
2019-05-27 11:58:28 INFO OGG-06511 Using following columns in default map by name: CUST_CODE, NAME, CITY, STATE.
2019-05-27 11:58:28 INFO OGG-06510 Using the following key columns for target table QASOURCE.TCUSTMER: CUST_CODE.
2019-05-27 11:58:28 INFO OGG-06506 Wildcard MAP resolved (entry QASOURCE.*): MAP “QASOURCE”.”TCUSTORD”, TARGET QASOURCE.”TCUSTORD”.
2019-05-27 11:58:28 INFO OGG-02756 The definition for table QASOURCE.TCUSTORD is obtained from the trail file.
2019-05-27 11:58:28 INFO OGG-06511 Using following columns in default map by name: CUST_CODE, ORDER_DATE, PRODUCT_CODE, ORDER_ID, PRODUCT_PRICE, PRODUCT_AMOUNT, TRANSACTION_ID.
2019-05-27 11:58:28 INFO OGG-06510 Using the following key columns for target table QASOURCE.TCUSTORD: CUST_CODE, ORDER_DATE, PRODUCT_CODE, ORDER_ID.
2019-05-27 11:58:56 INFO OGG-01021 Command received from GGSCI: STATS.
Issue1:
2019-05-27 09:58:32 ERROR OGG-15051 Java or JNI exception:
oracle.goldengate.util.GGException: org.springframework.beans.factory.BeanCreationException: Error creating bean with name ‘dataSourceConfig’ defined in class path resource [config/ggue-context.xml]: Bean
instantiation via constructor failed; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [oracle.goldengate.datasource.DataSourceConfig]: Constructor threw ex
ception; nested exception is oracle.goldengate.util.ConfigException: Unable to add to classpath, gg.classpath=/hadoop/hadoop-2.9.2/share/hadoop/common/*:/hadoop/hadoop-2.9.2/share/hadoop/common/lib/*:/had
oop/hadoop-2.9.2/share/hadoop/hdfs/*:/hadoop/hadoop-2.9.2/share/hadoop/hdfs/lib/*:/hadoop/hadoop-2.9.2/etc/hadoop/: (Additional runtime classpath setting) => Classpath directory does not exist: ‘/hadoop/h
adoop-2.9.2/share/hadoop/common/’
org.springframework.beans.factory.BeanCreationException: Error creating bean with name ‘dataSourceConfig’ defined in class path resource [config/ggue-context.xml]: Bean instantiation via constructor faile
d; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [oracle.goldengate.datasource.DataSourceConfig]: Constructor threw exception; nested exception is oracle.
goldengate.util.ConfigException: Unable to add to classpath, gg.classpath=/hadoop/hadoop-2.9.2/share/hadoop/common/*:/hadoop/hadoop-2.9.2/share/hadoop/common/lib/*:/hadoop/hadoop-2.9.2/share/hadoop/hdfs/*
:/hadoop/hadoop-2.9.2/share/hadoop/hdfs/lib/*:/hadoop/hadoop-2.9.2/etc/hadoop/: (Additional runtime classpath setting) => Classpath directory does not exist: ‘/hadoop/hadoop-2.9.2/share/hadoop/common/’
at oracle.goldengate.datasource.DataSourceLauncher.makeContext(DataSourceLauncher.java:270)
at oracle.goldengate.datasource.DataSourceLauncher.<init>(DataSourceLauncher.java:126)
at oracle.goldengate.datasource.UserExitMain.main(UserExitMain.java:108)
Caused by: org.springframework.beans.BeanInstantiationException: Failed to instantiate [oracle.goldengate.datasource.DataSourceConfig]: Constructor threw exception; nested exception is oracle.goldengate.u
til.ConfigException: Unable to add to classpath, gg.classpath=/hadoop/hadoop-2.9.2/share/hadoop/common/*:/hadoop/hadoop-2.9.2/share/hadoop/common/lib/*:/hadoop/hadoop-2.9.2/share/hadoop/hdfs/*:/hadoop/had
oop-2.9.2/share/hadoop/hdfs/lib/*:/hadoop/hadoop-2.9.2/etc/hadoop/: (Additional runtime classpath setting) => Classpath directory does not exist: ‘/hadoop/hadoop-2.9.2/share/hadoop/common/’
at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:163)
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:122)
at org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:267)
… 14 more
Caused by: oracle.goldengate.util.ConfigException: Unable to add to classpath, gg.classpath=/hadoop/hadoop-2.9.2/share/hadoop/common/*:/hadoop/hadoop-2.9.2/share/hadoop/common/lib/*:/hadoop/hadoop-2.9.2/s
hare/hadoop/hdfs/*:/hadoop/hadoop-2.9.2/share/hadoop/hdfs/lib/*:/hadoop/hadoop-2.9.2/etc/hadoop/: (Additional runtime classpath setting) => Classpath directory does not exist: ‘/hadoop/hadoop-2.9.2/share/
hadoop/common/’
at oracle.goldengate.datasource.DataSourceConfig.<init>(DataSourceConfig.java:611)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:147)
… 16 more
Caused by: java.io.IOException: Classpath directory does not exist: ‘/hadoop/hadoop-2.9.2/share/hadoop/common/’
at oracle.goldengate.util.ConfigClassPath.addJarsInFoldersToSystemClassPath(ConfigClassPath.java:148)
at oracle.goldengate.util.ConfigClassPath.addToClassPath(ConfigClassPath.java:216)
at oracle.goldengate.datasource.DataSourceConfig.<init>(DataSourceConfig.java:604)
… 21 more.
2019-05-27 09:58:32 ERROR OGG-01668 PROCESS ABENDING.
Issue2:
2019-05-27 10:03:06 INFO OGG-06506 Wildcard MAP resolved (entry QASOURCE.*): MAP “QASOURCE”.”TCUSTORD”, TARGET QASOURCE.”TCUSTORD”.
2019-05-27 10:03:06 INFO OGG-02756 The definition for table QASOURCE.TCUSTORD is obtained from the trail file.
2019-05-27 10:03:06 INFO OGG-06511 Using following columns in default map by name: CUST_CODE, ORDER_DATE, PRODUCT_CODE, ORDER_ID, PRODUCT_PRICE, PRODUCT_AMOUNT, TRANSACTION_ID.
2019-05-27 10:03:06 INFO OGG-06510 Using the following key columns for target table QASOURCE.TCUSTORD: CUST_CODE, ORDER_DATE, PRODUCT_CODE, ORDER_ID.
=ERROR 2019-05-27 10:03:07.000443 [main] – Failed to create the HDFS output stream to file [/datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTMER_RHDFS_2019-05-27_10-03-06.421.txt]
java.lang.RuntimeException: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.adl.AdlFileSystem not found
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2395) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3208) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3240) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:121) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3291) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3259) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:470) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:223) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:454) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356) ~[hadoop-common-2.9.2.jar:?]
at oracle.goldengate.handler.hdfs.HDFSFileWriter.open(HDFSFileWriter.java:138) [gghdfs-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.handler.hdfs.HDFSFileWriter.write(HDFSFileWriter.java:42) [gghdfs-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.handler.hdfs.HDFSHandler.processOp(HDFSHandler.java:317) [gghdfs-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.handler.hdfs.HDFSHandler.transactionCommit(HDFSHandler.java:225) [gghdfs-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.DsEventManager$2.send(DsEventManager.java:322) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.DsEventManager.distributeEvent(DsEventManager.java:110) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.DsEventManager.fireTransactionCommit(DsEventManager.java:330) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.AbstractDataSource.fireTransactionCommit(AbstractDataSource.java:511) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.UserExitDataSource.commitTransaction(UserExitDataSource.java:2000) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
at oracle.goldengate.datasource.UserExitDataSource.commitTx(UserExitDataSource.java:1690) [ggdbutil-12.3.2.1.0.018.jar:12.3.2.1.0.018]
Caused by: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.adl.AdlFileSystem not found
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2299) ~[hadoop-common-2.9.2.jar:?]
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2393) ~[hadoop-common-2.9.2.jar:?]
… 19 more
=ERROR 2019-05-27 10:03:07.000451 [main] – HDFS Handler failed to write to HDFS for operation: table=[QASOURCE.TCUSTMER], op pos=00000000000000001956, tx pos=00000000000000001956, op ts=2015-11-05 18:45:3
6.000000
Exception in thread “main” oracle.goldengate.util.GGException: Error detected handling transaction commit event.
at oracle.goldengate.datasource.UserExitDataSource.commitTransaction(UserExitDataSource.java:2002)
at oracle.goldengate.datasource.UserExitDataSource.commitTx(UserExitDataSource.java:1690)
Source Context :
SourceModule : [gglib.ggdal.adapter.java]
SourceID : [/scratch/aime/adestore/views/aime_adc4150505/oggcore/OpenSys/src/gglib/ggdal/Adapter/Java/JavaAdapter.cpp]
SourceMethod : [HandleJavaException]
SourceLine : [246]
ThreadBacktrace : [20] elements
: [/u01/app/ogg/libgglog.so(CMessageContext::AddThreadContext()+0x1e) [0x7f71fcb700ae]]
: [/u01/app/ogg/libgglog.so(CMessageFactory::CreateMessage(CSourceContext*, unsigned int, …)+0x6ac) [0x7f71fcb609bc]]
: [/u01/app/ogg/libgglog.so(_MSG_String(CSourceContext*, int, char const*, CMessageFactory::MessageDisposition)+0x39) [0x7f71fcb4ed19]]
: [/u01/app/ogg/libggjava.so(+0x2e9a7) [0x7f71eed9f9a7]]
: [/u01/app/ogg/libggjava.so(ggs::gglib::ggdal::CJavaRecordWriter::CommitTransaction(ggs::gglib::ggdal::CDALError&)+0x13d) [0x7f71eeda33ed]]
: [/u01/app/ogg/replicat(ggs::gglib::ggdal::CDALRecordWriter::CommitTransaction(ggs::gglib::ggdal::CDALError&)+0x4d) [0x82e92d]]
: [/u01/app/ogg/replicat(GenericImpl::CommitTransaction()+0x28) [0x8162e8]]
: [/u01/app/ogg/replicat(odbc_commit_transaction()+0x16) [0x80dbf6]]
: [/u01/app/ogg/replicat(Commit_and_Checkpoint(short, bool&, RepCsn&)+0x3d5) [0x6e3115]]
: [/u01/app/ogg/replicat(ClassicWorker::CommitAndCheckpoint(short)+0x2e) [0x5b6dde]]
: [/u01/app/ogg/replicat(REP_end_transaction(short)+0x17) [0x6e2a57]]
: [/u01/app/ogg/replicat(ggs::er::ReplicatContext::processReplicatLoop()+0x4897) [0x631157]]
: [/u01/app/ogg/replicat(ggs::er::ReplicatContext::run()+0x45) [0x60ff85]]
: [/u01/app/ogg/replicat() [0x6e4aaf]]
: [/u01/app/ogg/replicat(ggs::gglib::MultiThreading::MainThread::ExecMain()+0x5e) [0x7e27ce]]
: [/u01/app/ogg/replicat(ggs::gglib::MultiThreading::Thread::RunThread(ggs::gglib::MultiThreading::Thread::ThreadArgs*)+0x173) [0x7e6b93]]
: [/u01/app/ogg/replicat(ggs::gglib::MultiThreading::MainThread::Run(int, char**)+0x140) [0x7e7400]]
: [/u01/app/ogg/replicat(main+0x3b) [0x6e784b]]
: [/lib64/libc.so.6(__libc_start_main+0xf5) [0x7f71f52013d5]]
: [/u01/app/ogg/replicat() [0x550271]]
2019-05-27 10:03:07 ERROR OGG-15051 Java or JNI exception:
oracle.goldengate.util.GGException: Error detected handling transaction commit event.
***********************************************************************
* ** Run Time Statistics ** *
***********************************************************************
Reading /u01/app/ogg/dirdat/tr000000000, current RBA 3106, 4 records, m_file_seqno = 0, m_file_rba = 3106
#Sample gg.classpath for Apache Hadoop
gg.classpath=/u02/ogg/hadoop/hadoop-2.9.2/share/hadoop/common/*:/u02/ogg/hadoop/hadoop-2.9.2/share/hadoop/common/lib/*:/u02/ogg/hadoop/hadoop-2.9.2/share/hadoop/hdfs/*:/u02/ogg/hadoop/hadoop-2.9.2/share/h
adoop/hdfs/lib/*:/u02/ogg/hadoop/hadoop-2.9.2/etc/hadoop:/u02/ogg/hadoop/hadoop-2.9.2/share/hadoop/tools/lib/*
more hdfs.props
gg.handlerlist=hdfs
gg.handler.hdfs.type=hdfs
gg.handler.hdfs.includeTokens=false
gg.handler.hdfs.maxFileSize=1g
gg.handler.hdfs.pathMappingTemplate=/datalake-poc/raw/apmt_tos/Mobile
gg.handler.hdfs.fileRollInterval=0
gg.handler.hdfs.inactivityRollInterval=0
gg.handler.hdfs.fileNameMappingTemplate=${fullyQualifiedTableName}_${groupName}_${currentTimestamp}.txt
gg.handler.hdfs.partitionByTable=true
gg.handler.hdfs.rollOnMetadataChange=true
gg.handler.hdfs.authType=none
#gg.handler.hdfs.format=delimitedtext
gg.handler.hdfs.format=json
#gg.handler.hdfs.format.includeColumnNames=true
#gg.handler.hdfs.format.keyValueDelimiter=CDATA[=]
#gg.handler.hdfs.format.keyValuePairDelimiter=CDATA[,]
[oracle@linuxkit-025000000001 hadoop]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -cat /datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTMER_RHDFS_2019-05-27_14-51-10.358.txt
{“table”:”QASOURCE.TCUSTMER”,”op_type”:”I”,”op_ts”:”2015-11-05 18:45:36.000000″,”current_ts”:”2019-05-27T14:51:10.346000″,”pos”:”00000000000000001956″,”after”:{“CUST_CODE”:”WILL”,”NAME”:”BG SOFTWARE CO.”,”CITY”:”SEATTLE”,”STATE”:”WA”}}{“table”:”QASOURCE.TCUSTMER”,”op_type”:”I”,”op_ts”:”2015-11-05 18:45:36.000000″,”current_ts”:”2019-05-27T14:51:13.475000″,”pos”:”00000000000000002126″,”after”:{“CUST_CODE”:”JANE”,”NAME”:”ROCKY FLYER INC.”,”CITY”:”DENVER”,”STATE”:”CO”}}{“table”:”QASOURCE.TCUSTMER”,”op_type”:”I”,”op_ts”:”2015-11-05 18:45:39.000000″,”current_ts”:”2019-05-27T14:51:13.855000″,”pos”:”00000000000000003286″,”after”:{“CUST_CODE”:”DAVE”,”NAME”:”DAVE’S PLANES INC.”,”CITY”:”TALLAHASSEE”,”STATE”:”FL”}}{“table”:”QASOURCE.TCUSTMER”,”op_type”:”I”,”op_ts”:”2015-11-05 18:45:39.000000″,”current_ts”:”2019-05-27T14:51:13.855001″,”pos”:”00000000000000003462″,”after”:{“CUST_CODE”:”BILL”,”NAME”:”BILL’S USED CARS”,”CITY”:”DENVER”,”STATE”:”CO”}}{“table”:”QASOURCE.TCUSTMER”,”op_type”:”I”,”op_ts”:”2015-11-05 18:45:39.000000″,”current_ts”:”2019-05-27T14:51:13.855002″,”pos”:”00000000000000003600″,”after”:{“CUST_CODE”:”ANN”,”NAME”:”ANN’S BOATS”,”CITY”:”SEATTLE”,”STATE”:”WA”}}{“table”:”QASOURCE.TCUSTMER”,”op_type”:”U”,”op_ts”:”2015-11-05 18:45:39.000000″,”current_ts”:”2019-05-27T14:51:13.858001″,”pos”:”00000000000000005100″,”before”:{“CUST_CODE”:”ANN”,”NAME”:”ANN’S BOATS”,”CITY”:”SEATTLE”,”STATE”:”WA”},”after”:{“CUST_CODE”:”ANN”,”CITY”:”NEW YORK”,”STATE”:”NY”}}[
gg.handlerlist=hdfs
gg.handler.hdfs.type=hdfs
gg.handler.hdfs.includeTokens=false
gg.handler.hdfs.maxFileSize=1g
gg.handler.hdfs.pathMappingTemplate= datalake-poc/raw/apmt_tos/Mobile
gg.handler.hdfs.fileRollInterval=0
gg.handler.hdfs.inactivityRollInterval=0
gg.handler.hdfs.fileNameMappingTemplate=${fullyQualifiedTableName}_${groupName}_${currentTimestamp}.txt
gg.handler.hdfs.partitionByTable=true
gg.handler.hdfs.rollOnMetadataChange=true
gg.handler.hdfs.authType=none
gg.handler.hdfs.format=delimitedtext
gg.handler.hdfs.format.includeColumnNames=true
gg.handler.hdfs.mode=tx
#gg.handler.hdfs.format.includeOpTimestamp=true
#gg.handler.hdfs.format.includeCurrentTimestamp=true
#gg.handler.hdfs.format.insertOpKey=I
#gg.handler.hdfs.format.updateOpKey=U
#gg.handler.hdfs.format.deleteOpKey=D
#gg.handler.hdfs.format.truncateOpKey=T
#gg.handler.hdfs.format.encoding=UTF-8
gg.handler.hdfs.format.fieldDelimiter=CDATA[\u0001]
gg.handler.hdfs.format.lineDelimiter=CDATA[\n]
gg.handler.hdfs.format.includeTableName=true
gg.handler.hdfs.format.keyValueDelimiter=CDATA[=]
gg.handler.hdfs.format.kevValuePairDelimiter=CDATA[;]
#gg.handler.hdfs.format.pkUpdateHandling=abend
#gg.handler.hdfs.format.nullValueRepresentation=NULL
#gg.handler.hdfs.format.missingValueRepresentation=CDATA[]
#gg.handler.hdfs.format.includePosition=true
#gg.handler.hdfs.format=delimitedtext
Caused by: oracle.goldengate.util.ConfigException: Failed to set property: gg.handler.hdfs.format.kevValuePairDelimiter:=”CDATA[,]” (class: oracle.goldengate.format.text.DelimitedTextFormatter).
GGSCI (linuxkit-025000000001) 5> start replicat rhdfs NOFILTERDUPTRANSACTIONS
[oracle@linuxkit-025000000001 hadoop]$ /u02/ogg/hadoop/hadoop-2.9.2/bin/hadoop fs -cat /datalake-poc/raw/apmt_tos/Mobile/QASOURCE.TCUSTMER_RHDFS_2019-05-27_16-08-24.868.txt
IQASOURCE.TCUSTMER2015-11-05 18:45:36.0000002019-05-27T16:08:24.85500000000000000000001956CUST_CODEWILLNAMEBG SOFTWARE CO.CITYSEATTLESTATEWA
IQASOURCE.TCUSTMER2015-11-05 18:45:36.0000002019-05-27T16:08:28.23000000000000000000002126CUST_CODEJANENAMEROCKY FLYER INC.CITYDENVERSTATECO
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T16:08:28.34500000000000000000003286CUST_CODEDAVENAMEDAVE’S PLANES INC.CITYTALLAHASSEESTATEFL
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T16:08:28.34800000000000000000003462CUST_CODEBILLNAMEBILL’S USED CARSCITYDENVERSTATECO
IQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T16:08:28.34900000000000000000003600CUST_CODEANNNAMEANN’S BOATSCITYSEATTLESTATEWA
UQASOURCE.TCUSTMER2015-11-05 18:45:39.0000002019-05-27T16:08:28.35100200000000000000005100CUST_CODEANNNAMECITYNEW YORKSTATENY
