Monday, September 23, 2019

Powershell: Generating random password

Here's method for generating random password by using Powershell.

# Add the System.Web assembly
PS C:\Users\drago> Add-Type -AssemblyName System.Web # Let's get random password by following method
# First parameter is length for the password, and second is the number of non-alphabetical characters.
PS C:\Users\drago> [System.Web.Security.Membership]::GeneratePassword(10,4)
.Ra45;)WQ}

Monday, September 16, 2019

docker: [RESOLVED] Expose dockerised mongodb listenport, and connection failed :(

I created dockerised mongodb, but encountered login failure via mongo shell.
An investigation found that my depreciated mongo shell version caused issue.
After that, I updated the software version, and remote connection got successful!

#prepare persistent data location with docker volume
docker volume create my-vol
docker volume inspect my-vol
[
    {
        "CreatedAt": "2019-09-15T01:24:25+09:00",
        "Driver": "local",
        "Labels": {},
        "Mountpoint": "/var/lib/docker/volumes/my-vol/_data",
        "Name": "my-vol",
        "Options": {},
        "Scope": "local"
    }
]
# Copy dumped database file for restoration
cp -R ./testCol /var/lib/docker/volumes/my-vol/_data


# Run mongo container mounting created volume, and specify mongodb data directory as /data/db
docker run --name mongo-dev -d -v my-vol:/data/db  -e DATA_DIR=/data/db -p 27017 mongo
d2b759805a018d1a71b835c7d6dffdda68cbbfa20f7f891fb381a53fd7b49ed0


# Restore database from dumpfile
docker exec -it mongo-dev mongorestore --db articles /data/db/articles


# Check local port
docker ps 
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                      NAMES
d2b759805a01        mongo               "docker-entrypoint.s…"   23 seconds ago      Up 21 seconds       0.0.0.0:32768->27017/tcp   mongo-dev


# Try to connect mongo console, but it fails
mongo --host 127.0.0.1:32768
MongoDB shell version v3.4.21
connecting to: mongodb://127.0.0.1:32768/
2019-09-15T23:47:20.861+0900 E QUERY    [thread1] Error: network error while attempting to run command 'whatsmyuri' on host '127.0.0.1:32768'  :
connect@src/mongo/shell/mongo.js:240:13
@(connect):1:6
exception: connect failed


# Check listen status, dockerised mongo is bound to ipv6 port 32768
netstat -ltn
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN
tcp6       0      0 ::1:25                  :::*                    LISTEN
tcp6       0      0 :::32768                :::*                    LISTEN
tcp6       0      0 :::22                   :::*                    LISTEN


# Check container with interactive console
docker run -it --link=mongo-dev:mongo mongo bash


# I can see mongodb collections via console
mongo --host 172.17.0.2:27017
> show databases;
admin          0.000GB
articles       0.000GB
config         0.000GB
local          0.000GB
sepfourteenth  0.000GB
> use articles;
switched to db articles
> db.testCol.find({});
{ "_id" : ObjectId("5d6682061f2af8077f940255"), "name" : "大丈夫", "query" : "%E5%A4%A7%E4%B8%88%E5%A4%AB", "tweet_volume" : 490680, "date" : "8/28/2019, 10:30:45 PM" }
{ "_id" : ObjectId("5d6682061f2af8077f940256"), "name" : "テレビ", "query" : "%E3%83%86%E3%83%AC%E3%83%93", "tweet_volume" : 337330, "date" : "8/28/2019, 10:30:45 PM" }


# Let me specify source IP and port
docker run --name mongo-dev -d -v my-vol:/data/db  -e DATA_DIR=/data/db -p 127.0.0.1:32768:27017  mongo --bind_ip_all
a3f9ab22beaed1b13e73f3ab0d4ccd452a8f1b4c33688c7d0906acd0d2f264ed


# Now I can see the listen port bind to ipv4
netstat -ltn
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN
tcp        0      0 127.0.0.1:32768         0.0.0.0:*               LISTEN
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN
tcp6       0      0 ::1:25                  :::*                    LISTEN
tcp6       0      0 :::22                   :::*                    LISTEN

docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                        NAMES
a3f9ab22beae        mongo               "docker-entrypoint.s…"   42 seconds ago      Up 41 seconds       127.0.0.1:32768->27017/tcp   mongo-dev


# Unfortunately another issue was found when I tried to connect dockerised mongo.
mongo --host 127.0.0.1:32768
MongoDB shell version v3.4.21
connecting to: mongodb://127.0.0.1:32768/
2019-09-16T20:15:42.149+0900 E QUERY    [thread1] Error: network error while attempting to run command 'whatsmyuri' on host '127.0.0.1:32768'  :
connect@src/mongo/shell/mongo.js:240:13
@(connect):1:6
exception: connect failed


# bind_ip_all was specified.
docker container inspect mongo-dev | grep -n -A10 bind
7:            "--bind_ip_all"
8-        ],
9-        "State": {
10-            "Status": "running",
11-            "Running": true,
12-            "Paused": false,
13-            "Restarting": false,
14-            "OOMKilled": false,
15-            "Dead": false,
16-            "Pid": 1635,
17-            "ExitCode": 0,
--
193:                "--bind_ip_all"
194-            ],


# Unfamiliar error was written on log file.
I realized the problem sits on host side.

{"log":"2019-09-16T15:19:20.315+0000 E  -        [conn2] Assertion: Location34348: cannot translate opcode 2010 src/mongo/rpc/message.h 120\n","stream":"stdout","time":"2019-09-16T15:19:20.315419954Z"}



# Checked mongo shell version on host.
mongo --version


# Update yum repository file for fetching newer version of mongo shell.
yum remove mongodb-org-shell
vi /etc/yum.repos.d/mongodb-org.repo

name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.0/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc


# Reintall mongo shell.
yum remove mongodb-org-shell
yum install mongodb-org-shell


# Now everything alright! :)
The trouble shooting was really tough though...(ToT)

mongo 127.0.0.1:32768                                 MongoDB shell version v4.0.12
connecting to: mongodb://127.0.0.1:32768/test?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6e2290f5-c297-4c81-8c25-c1197311c8d9") }
MongoDB server version: 4.2.0

> show databases;
admin          0.000GB
articles       0.000GB
config         0.000GB
local          0.000GB
sepfourteenth  0.000GB
> exit
bye

Sunday, September 8, 2019

docker: Pull mongo image and restore backuped database to container

Let's see how I tried to restore local mongo database to mongo docker container.
'docker cp' command didn't work for host -> container file copy.
I created docker volume and used it to achieve this.

Retrieve mongo image from docker repository
[root@localhost ~]# docker pull mongo

Backup existing database with 'mongodump'
[root@localhost ~]# mongodump --collection testCol --db articles --out ./testCol

Let's create docker volume for passing backup files to docker container
[root@localhost ~]# docker volume create my-vol

Check the full path of created volume
[root@localhost ~]# docker volume inspect my-vol
[
    {
        "CreatedAt": "2019-09-05T00:20:34+09:00",
        "Driver": "local",
        "Labels": {},
        "Mountpoint": "/var/lib/docker/volumes/my-vol/_data",
        "Name": "my-vol",
        "Options": {},
        "Scope": "local"
    }
]

Copy back up files to the volume, so that we can access them from container later
[root@localhost ~]# cp -R ./testCol/* /var/lib/docker/volumes/my-vol/_data/

Run container with mounting created volume
[root@localhost ~]# docker run -itd --mount source=my-vol,target=/app mongo

Identify the container name
[root@localhost ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d70f1c3f0101 mongo "docker-entrypoint.s…" 5 seconds ago Up 3 seconds 27017/tcp admiring_bell

Verify if the volume is being mounted properly
[root@localhost ~]# docker inspect admiring_bell | grep -A10 -i mount
            "Mounts": [
                {
                    "Type": "volume",
                    "Source": "my-vol",
                    "Target": "/app"
                }
            ],
            "MaskedPaths": [
                "/proc/asound",
                "/proc/acpi",
                "/proc/kcore",
--
        "Mounts": [
            {
                "Type": "volume",
                "Name": "my-vol",
                "Source": "/var/lib/docker/volumes/my-vol/_data",
                "Destination": "/app",
                "Driver": "local",
                "Mode": "z",
                "RW": true,
                "Propagation": ""
            },


Now we can see copied directory from the container
[root@localhost ~]# docker exec -it admiring_bell ls -l /app
total 0
drwxr-xr-x. 2 root root 55 Sep 4 15:20 articles

Restore back up files to database inside the container
[root@localhost ~]# docker exec -it admiring_bell mongorestore --db articles /app/articles
2019-09-08T05:51:16.952+0000 the --db and --collection args should only be used when restoring from a BSON file. Other uses are deprecated and will not exist in the future; use --nsInclude instead
2019-09-08T05:51:16.953+0000 building a list of collections to restore from /app/articles dir
2019-09-08T05:51:16.956+0000 reading metadata for articles.testCol from /app/articles/testCol.metadata.json
2019-09-08T05:51:17.005+0000 restoring articles.testCol from /app/articles/testCol.bson
2019-09-08T05:51:17.022+0000 no indexes to restore
2019-09-08T05:51:17.023+0000 finished restoring articles.testCol (38 documents, 0 failures)
2019-09-08T05:51:17.023+0000 38 document(s) restored successfully. 0 document(s) failed to restore.


Let's check the restored database with interactive console of the container
[root@localhost ~]# docker run -it --link=admiring_bell:mongo mongo /bin/sh

# env
HOSTNAME=d74d7531ee8c
~~~~~~~
MONGO_PORT_27017_TCP=tcp://172.17.0.2:27017
~~~~~~~

# mongo --host 172.17.0.2:27017
MongoDB shell version v4.2.0
connecting to: mongodb://172.17.0.2:27017/?compressors=disabled&gssapiServiceName=mongodb

> show databases
admin 0.000GB
articles 0.000GB
config 0.000GB
local 0.000GB

> use articles

switched to db articles

> show collections
testCol

> db.testCol.find({})
{ "_id" : ObjectId("5d6682061f2af8077f940255"), "name" : "大丈夫", "query" : "%E5%A4%A7%E4%B8%88%E5%A4%AB", "tweet_volume" : 490680, "date" : "8/28/2019, 10:30:45 PM" }
{ "_id" : ObjectId("5d6683097633bc07a7b6f03d"), "name" : "大丈夫", "query" : "%E5%A4%A7%E4%B8%88%E5%A4%AB", "tweet_volume" : 493621, "date" : "8/28/2019, 10:35:04 PM" }
Type "it" for more

> exit


Everything works fine, now we can stop the container
[root@localhost ~]# docker container stop admiring_bell
admiring_bell

Clean it up
[root@localhost ~]# docker system prune