Problem with backup cbbackupmgr


#1

Hello all,

Have you ever encountered this problem ?

root@BDD-P1:/data/backup-couchbase/prod#
/var/lib/docker/aufs/diff/45ab5f4212d3064ecaca899c75723fae66357bcd18898b275cb4f689820f1ed5/opt/couchbase/bin/cbbackupmgr backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Admin --password xxxx
root@BDD-P1:/data/backup-couchbase/prod#

root@BDD-P1:/data/backup-couchbase/prod# /var/lib/docker/aufs/diff/45ab5f4212d3064ecaca899c75723fae66357bcd18898b275cb4f689820f1ed5/opt/couchbase/bin/cbbackupmgr backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Admin --password xxxx --resume
root@BDD-P1:/data/backup-couchbase/prod#

root@BDD-P1:/data/backup-couchbase/prod# /var/lib/docker/aufs/diff/45ab5f4212d3064ecaca899c75723fae66357bcd18898b275cb4f689820f1ed5/opt/couchbase/bin/cbbackupmgr backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Admin --password xxxx --purge
root@BDD-P1:/data/backup-couchbase/prod#

in logs :

root@BDD-P1:/data/backup-couchbase/logs# tail -f backup.log
net/http.(*persistConn).readLoop(0xc8200f60b0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:876 +0xf7
created by net/http.(*Transport).dialConn
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:685 +0xc78

goroutine 19 [select]:
net/http.(*persistConn).writeLoop(0xc8200f60b0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:1009 +0x40c
created by net/http.(*Transport).dialConn
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:686 +0xc9d

impossible to make a backup


#2

Hello @ssa,

Sorry to hear you’re having a problem with cbbackupmgr. Would it be possible to share the whole log please?

Thanks,
Patrick


#3

Thanks for your help Patrick.

2018-10-18T08:27:36.763+02:00 [INFO][FDB] Forestdb opened database file /data/backup-couchbase/prod/2018-08-18T13_00_01.280319606+02_00/venus-data-35a4a23fc53b6972d2b0a336803f545c/data/shard_0.fdb
2018-10-18T08:27:36.766+02:00 [INFO][FDB] Forestdb closed database file /data/backup-couchbase/prod/2018-08-18T13_00_01.280319606+02_00/venus-data-35a4a23fc53b6972d2b0a336803f545c/data/shard_0.fdb
2018-10-18T08:27:36.766+02:00 [INFO][FDB] Forestdb opened database file /data/backup-couchbase/prod/2018-08-18T13_00_01.280319606+02_00/venus-ged-8c21794e75049b55fdc52932e5e12892/data/shard_0.fdb
2018-10-18T08:27:36.768+02:00 [INFO][FDB] Forestdb closed database file /data/backup-couchbase/prod/2018-08-18T13_00_01.280319606+02_00/venus-ged-8c21794e75049b55fdc52932e5e12892/data/shard_0.fdb
2018-10-18T08:27:36.769+02:00 (Cmd) Archive contents listed successfully
2018-10-18T08:28:28.120+02:00 (Cmd) backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Administrator --password ********
2018-10-18T08:28:28.122+02:00 (Rest) GET http://127.0.0.1:8091/pools 200
panic: interface conversion: interface is nil, not string

goroutine 1 [running]:
github.com/couchbase/backup/plan.parseSerial(0xc820118e70, 0x6)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:29 +0x2a7e
github.com/couchbase/backup/plan.parseParallel(0xc820118e40, 0x8)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/parallel.go:47 +0x2019
github.com/couchbase/backup/plan.parseSerial(0xc820118d80, 0xc820118d50)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:41 +0x2208
github.com/couchbase/backup/plan.ParsePlan(0x7a57a0, 0xc820118d50, 0x787f00)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/plan.go:90 +0xa1
github.com/couchbase/backup/archive.(*incrBackupDir).readPlan(0xc8200fe2d0, 0xc8200fe2d0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/dir_incr.go:159 +0xcc
github.com/couchbase/backup/archive.(*Archive).CreateIncrBackup(0xc8200db410, 0x7fff438de6f7, 0x4, 0x7f6d192b0000, 0x7f6d192bd100, 0xc820112180, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/archive.go:114 +0x139
github.com/couchbase/backup.CouchbaseToArchiveTransferable(0xc8200db410, 0x7fff438de6f7, 0x4, 0xc8200a0580, 0x15, 0x7fff438de724, 0xd, 0x7fff438de73d, 0x7, 0x0, …)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/transferable.go:74 +0x7c0
main.parseBackupCommand()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/backup.go:104 +0xafb
main.main()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/main.go:86 +0x613

goroutine 17 [syscall, locked to thread]:
runtime.goexit()

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 20 [syscall]:
os/signal.loop()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 21 [chan receive]:
main.sigHandler()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/main.go:307 +0x171
created by main.parseBackupCommand
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/backup.go:94 +0x94a

goroutine 34 [select, locked to thread]:
runtime.gopark(0x9d5250, 0xc820122728, 0x8fd728, 0x6, 0x447518, 0x2)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc820122728, 0x0, 0x18)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc820122728)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 23 [IO wait]:
net.runtime_pollWait(0x7f6d19302b60, 0x72, 0xc8200740a0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8200f8fb0, 0x72, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8200f8fb0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8200f8f50, 0xc82013a000, 0x1000, 0x1000, 0x0, 0x7f6d192fd028, 0xc8200740a0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc82007e128, 0xc82013a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7f6d19302c20, 0xc82007e128, 0xc8201001b8, 0xc82013a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:1370 +0x67
net/http.(*noteEOFReader).Read(0xc8200a0da0, 0xc82013a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
:126 +0xd0
bufio.(*Reader).fill(0xc8200737a0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc8200737a0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc820100160)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:876 +0xf7
created by net/http.(*Transport).dialConn
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:685 +0xc78

goroutine 24 [select]:
net/http.(*persistConn).writeLoop(0xc820100160)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:1009 +0x40c
created by net/http.(*Transport).dialConn
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:686 +0xc9d
2018-10-18T08:28:36.421+02:00 (Cmd) backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Administrator --password ******* --resume
2018-10-18T08:28:36.423+02:00 (Rest) GET http://127.0.0.1:8091/pools 200
panic: interface conversion: interface is nil, not string

goroutine 1 [running]:
github.com/couchbase/backup/plan.parseSerial(0xc820102e70, 0x6)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:29 +0x2a7e
github.com/couchbase/backup/plan.parseParallel(0xc820102e40, 0x8)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/parallel.go:47 +0x2019
github.com/couchbase/backup/plan.parseSerial(0xc820102d80, 0xc820102d50)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:41 +0x2208
github.com/couchbase/backup/plan.ParsePlan(0x7a57a0, 0xc820102d50, 0x787f00)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/plan.go:90 +0xa1
github.com/couchbase/backup/archive.(*incrBackupDir).readPlan(0xc8200e42d0, 0xc8200e42d0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/dir_incr.go:159 +0xcc
github.com/couchbase/backup/archive.(*Archive).CreateIncrBackup(0xc8200c1440, 0x7ffc956326ee, 0x4, 0x7fa9d8970100, 0x7fa9d897b100, 0xc8200fc180, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/archive.go:114 +0x139
github.com/couchbase/backup.CouchbaseToArchiveTransferable(0xc8200c1440, 0x7ffc956326ee, 0x4, 0xc820010680, 0x15, 0x7ffc9563271b, 0xd, 0x7ffc95632734, 0x7, 0x0, …)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/transferable.go:74 +0x7c0
main.parseBackupCommand()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/backup.go:104 +0xafb
main.main()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/main.go:86 +0x613

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 5 [syscall]:
os/signal.loop()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 6 [chan receive]:
main.sigHandler()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/main.go:307 +0x171
created by main.parseBackupCommand
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/backup.go:94 +0x94a

goroutine 8 [IO wait]:
net.runtime_pollWait(0x7fa9db430ac0, 0x72, 0xc82000c1d0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8200defb0, 0x72, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8200defb0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8200def50, 0xc8200e7000, 0x1000, 0x1000, 0x0, 0x7fa9db42b050, 0xc82000c1d0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc82002a130, 0xc8200e7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7fa9db430b80, 0xc82002a130, 0xc8200f6108, 0xc8200e7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:1370 +0x67
net/http.(*noteEOFReader).Read(0xc820010ea0, 0xc8200e7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
:126 +0xd0
bufio.(*Reader).fill(0xc820019800)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc820019800, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/bufio/bufio.go:132 +0xcc
net/http.(*persistConn).readLoop(0xc8200f60b0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:876 +0xf7
created by net/http.(*Transport).dialConn
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:685 +0xc78

goroutine 9 [select]:
net/http.(*persistConn).writeLoop(0xc8200f60b0)
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:1009 +0x40c
created by net/http.(*Transport).dialConn
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/net/http/transport.go:686 +0xc9d
2018-10-18T08:28:41.557+02:00 (Cmd) backup --archive /data/backup-couchbase --repo prod --host couchbase://127.0.0.1 --username Administrator --password ******* --purge
2018-10-18T08:28:41.562+02:00 (Rest) GET http://127.0.0.1:8091/pools 200
panic: interface conversion: interface is nil, not string

goroutine 1 [running]:
github.com/couchbase/backup/plan.parseSerial(0xc820132e10, 0x6)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:29 +0x2a7e
github.com/couchbase/backup/plan.parseParallel(0xc820132de0, 0x8)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/parallel.go:47 +0x2019
github.com/couchbase/backup/plan.parseSerial(0xc820132d20, 0xc820132cf0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/serial.go:41 +0x2208
github.com/couchbase/backup/plan.ParsePlan(0x7a57a0, 0xc820132cf0, 0x787f00)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/plan/plan.go:90 +0xa1
github.com/couchbase/backup/archive.(*incrBackupDir).readPlan(0xc8200e42d0, 0xc8200e42d0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/dir_incr.go:159 +0xcc
github.com/couchbase/backup/archive.(*Archive).CreateIncrBackup(0xc8200c1440, 0x7fff80d4c6ef, 0x4, 0x7f3bb0890001, 0x7f3bb089d100, 0xc82012c060, 0x0, 0x0, 0x0, 0x0)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/archive/archive.go:114 +0x139
github.com/couchbase/backup.CouchbaseToArchiveTransferable(0xc8200c1440, 0x7fff80d4c6ef, 0x4, 0xc820010680, 0x15, 0x7fff80d4c71c, 0xd, 0x7fff80d4c735, 0x7, 0x0, …)
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/transferable.go:74 +0x7c0
main.parseBackupCommand()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/backup.go:104 +0xafb
main.main()
/home/couchbase/jenkins/workspace/watson-unix/goproj/src/github.com/couchbase/backup/cmd/cbbackupmgr/main.go:86 +0x613

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.5.2/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 5 [syscall]:
os/signal.loop()


#4

What version of Couchbase Server is being used? It looks like it panics just after it gets the pools response.


#5

Version: “5.1.0-5552-enterprise”.