Migrating from Mongodb to Dynamo db

referernce: https://www.codementor.io/devops/tutorial/handling-date-and-datetime-in-dynamodb


Dynamo Db does not support multiple indexes
It supports only Hash key, hash key + range key only
Does not support native date, DateTime object

Advantage:
No worries of scaling, monitoring, security updates.
average good response.

Use unix time stamp, store as Number
Combine multiple mongo db key if required
Database denormalization
Based on query patter, create schema

Dynamo DB is  NO SQL db

Queries are slow
GetItem call as fast

Terminology
Table , Item, Attribute, Primary Key, hash attribute value, hash and range attribute values
secondary index on non key attibute value

Start the dynamo db locally
java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -help
Dynamo db port: 8000

Create table
var params = {
  TableName : "Music",
  KeySchema:[
    {AttributeName:"Artist", KeyType:"Hash"},
    {AttributeName:"SongTitle", KeyType:"Range"}
  ],
  AttributeDefinitions:[
     {AttibuteName:"Artist", AttibuteType:"S"},
     {AttibuteName:"SongTitle", AttibuteType:"S"},
  ],
  ProvisionedThroughput:{
      ReadCapacityUnits:1,
      WriteCapacityUnits: 1
  }
};
dynamodb.createTable(params, function(err, data){

});

TableStatus is should be Active

2)
var params = { TableName: "Music" }; dynamodb.describeTable(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

3)
var params = {}; dynamodb.listTables(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

4)

var params = { TableName: "Music", Item: { "Artist":"No One You Know", "SongTitle":"Call Me Today", "AlbumTitle":"Somewhat Famous", "Year": 2015, "Price": 2.14, "Genre": "Country", "Tags": { "Composers": [ "Smith", "Jones", "Davis" ], "LengthInSeconds": 214 } } }; dynamodb.putItem(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

primarykey is must, remaining attribute are optional

5)Condition Expressoin

var params = { TableName: "Music", Item: { "Artist":"No One You Know", "SongTitle":"Call Me Today", "AlbumTitle":"Somewhat Famous", "Year": 2015, "Price": 2.14, "Genre": "Country", "Tags": { "Composers": [ "Smith", "Jones", "Davis" ], "LengthInSeconds": 214 } }, "ConditionExpression": "attribute_not_exists(Artist) and attribute_not_exists(SongTitle)" };

6)Batch insert
var params = { RequestItems: { "Music": [ { PutRequest: { Item: { "Artist": "No One You Know", "SongTitle": "My Dog Spot", "AlbumTitle":"Hey Now", "Price": 1.98, "Genre": "Country", "CriticRating": 8.4 } } }, { PutRequest: { Item: { "Artist": "No One You Know", "SongTitle": "Somewhere Down The Road", "AlbumTitle":"Somewhat Famous", "Genre": "Country", "CriticRating": 8.4, "Year": 1984 } } }
]}

7)Dynamo db data types
String, number, float, map, list

8)
var params = { TableName: "Music", Key: { "Artist": "No One You Know", "SongTitle": "Call Me Today" } }; dynamodb.getItem(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });
9)
var params = { TableName: "Music", Key: { "Artist": "No One You Know", "SongTitle": "Call Me Today" }, ProjectionExpression: "AlbumTitle" };
10)
var params = { TableName: "Music", KeyConditionExpression: "Artist = :artist", ExpressionAttributeValues: { ":artist": "No One You Know" } }; dynamodb.query(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

11)

var params = { TableName: "Music" }; dynamodb.scan(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

Operations

12)

var params = { TableName: "Music", AttributeDefinitions:[ {AttributeName: "Genre", AttributeType: "S"}, {AttributeName: "Price", AttributeType: "N"} ], GlobalSecondaryIndexUpdates: [ { Create: { IndexName: "GenreAndPriceIndex", KeySchema: [ {AttributeName: "Genre", KeyType: "HASH"}, {AttributeName: "Price", KeyType: "RANGE"}, ], Projection: { "ProjectionType": "ALL" }, ProvisionedThroughput: { "ReadCapacityUnits": 1,"WriteCapacityUnits": 1 } } } ] }; dynamodb.updateTable(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

13)
var params = { TableName: "Music", Key: { "Artist":"No One You Know", "SongTitle":"Call Me Today" }, UpdateExpression: "SET RecordLabel = :label", ExpressionAttributeValues: { ":label": "Global Records" }, ReturnValues: "ALL_NEW" }; dynamodb.updateItem(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

15)Automatic counter

15)
var params = { TableName: "Music", Key: { Artist: "The Acme Band", SongTitle: "Look Out, World" } }; dynamodb.deleteItem(params, function(err, data) { if (err) console.log(JSON.stringify(err, null, 2)); else console.log(JSON.stringify(data, null, 2)); });

http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/BestPractices.html

refernce: http://docs.aws.amazon.com/amazondynamodb/latest/gettingstartedguide/GettingStarted.Java.html

Play Framework application structure

reference:https://www.playframework.com/documentation/2.0/Anatomy

app                      → Application sources
 └ assets                → Compiled asset sources
    └ stylesheets        → Typically LESS CSS sources
    └ javascripts        → Typically CoffeeScript sources
 └ controllers           → Application controllers
 └ models                → Application business layer
 └ views                 → Templates
conf                     → Configurations files
 └ application.conf      → Main configuration file
 └ routes                → Routes definition
public                   → Public assets
 └ stylesheets           → CSS files
 └ javascripts           → Javascript files
 └ images                → Image files
project                  → sbt configuration files
 └ build.properties      → Marker for sbt project
 └ Build.scala           → Application build script
 └ plugins.sbt           → sbt plugins
lib                      → Unmanaged libraries dependencies
logs                     → Standard logs folder
 └ application.log       → Default log file
target                   → Generated stuff
 └ scala-2.9.1              
    └ cache              
    └ classes            → Compiled class files
    └ classes_managed    → Managed class files (templates, ...)
    └ resource_managed   → Managed resources (less, ...)
    └ src_managed        → Generated sources (templates, ...)
test                     → source folder for unit or functional tests

Play Framework controller naming conventions

Singular, Noun.
Suffix: Controller

ex: UserController

Rest api naming conventions

Refernce: https://developers.google.com/drive/v2/reference/

using google drive as reference

1)Version should be part of rest api url even for get, search, insert, update, delete
ex: https://www.googleapis.com/upload/drive/v2/files 

2)HTTP methods usages
GET for read
POST for insert
PUT for update
DELETE for delete

ex:
#Get file meta data
GET /files/ 
#insert file
POST /files
#update file
PUT /files/
#delete file
DELETE /files/
note: place holder for fileid 
GET /files/234 here 234 is fileid

3)Use plural, nouns for resource name
ex: https://www.googleapis.com/upload/drive/v2/files 
files

andrid teaching Free goodies

andrid teaching Free goodies pen drive, 6k mobile, food, laptop share, bike share ,stayshare => 80k .5k+6k+3k+15k+15k+3k+ 10k => 50k

branded professional teacher hyperlocal small batch at teacher house/pg in person

branded , professional, teacher, hyper, local, in person, small batch, quality, 45 days, 1 hour a day, workshop, extra options, smal job , bpo, document collector, freelance job, part time job, delivery boy, food, accomodation, tranportation by auto, bike, car., hard material, projects every day, guru , traditional teaching settting , no fancy place, just q
---
Mayur will teach android including java for fresher
fee: 10k
every day : 1hour timing: 8am to 9am
sun day: 3 hour
place: his pg

php, codeigniter, html, css
fee: 6k
every day : 1hour timing: 8am to 9am
sun day: 3 hour
place: his pg

get pen drive
copy project for day
teaching for last  3 years

student should come to teacher
teacher will come very near by place
if student, teacher can be transported very fast by auto, bike, car is good
need place for teaching

two people  for teaching android
gopinath, mayur

supply student , such teacher, who are working , have less time to spare
student does not have internet,laptop, mobile

give loan of laptop, mobile, tution fee
laptop
mobile
tution fee
temporary  job bpo, operations, document collector
accomodation
food
------
shared laptop, mobile
reduce tution fee on mass booking,regular students
---
even provide job interview options
---
provide freelance work
---
branded profession teacher aggregator hyper local ,in person aggregator
professional teacher working on companies startup, big companies
--
filter the student within 1 week if can or not
if can switch to php, or operation
---
even provider temporary job: call support, operations
---
profession teacher to fresher
conduct workshop online, offline on sunday, saturday
---

What is Amazon EBS?

Elastic Block Storage
This is storage which lives independent of ec2 instance state
Its available only in one region

Need:
data change frequently
long term storage

Use
File system
Database
Messaging QuUE

RabbitMQ start, stop, status, log , data location

Start RabbitMQ
sudo rabbitmqctl start

Stop RabbitMQ
sudo rabbitmqctl stop

Status of RabbitMQ
sudo rabbitmqctl status

Log location:
/var/log/rabbitmq

Data location:
/var/lib/rabbitmq

Unable to access rabbitmq management website via guest/guest as rabbitmq hosted on ec2

reference: https://geekforum.wordpress.com/2014/05/02/install-run-and-configure-rabbitmq-on-aws-ec2/

#add admin user for RabbitMQ
sudo rabbitmqctl add_user admin admin

#Set pemission for admin user
sudo rabbitmqctl set_permissions -p / admin ".*" ".*" ".*"

#make admin as user administrator
sudo rabbitmqctl set_user_tags admin administrator

Use admin for login now

Integrate swagger with Play Framework link /practice in progress

http://swagger.io/playing-with-swagger-using-swagger-and-swagger-ui-with-the-play-framework/

i just read it, yet to check and practice in full

build.sbt
----
"com.wordnik" %% "swagger-play2" % "1.3.12" exclude("org.reflections", "reflections"), "org.reflections" % "reflections" % "0.9.8" notTransitive (), "org.webjars" % "swagger-ui" % "2.1.8-M1"

application.conf :
----
api.version="1.0" swagger.api.basepath="http://localhost:9000"

routes
----
GET / controllers.Assets.at(path="/public", file="index.html")

GET /api-docs controllers.ApiHelpController.getResources

POST /login controllers.SecurityController.login() POST /logout controllers.SecurityController.logout()

GET /api-docs/api/todos controllers.ApiHelpController.getResource(path = "/api/todos")
GET /todos controllers.TodoController.getAllTodos()
POST /todos controllers.TodoController.createTodo()


swagger annotation
--------
@Api(value = "/api/todos", description = "Operations with Todos") @Security.Authenticated(Secured.class)
public class TodoController extends Controller {


@ApiOperation(value = "get All Todos",
     notes = "Returns List of all Todos",
     response = Todo.class,
     httpMethod = "GET")
public static Result getAllTodos() {
     return ok(toJson(models.Todo.findByUser(SecurityController.getUser())));
}
@ApiOperation(
     nickname = "createTodo",
     value = "Create Todo",
     notes = "Create Todo record",
     httpMethod = "POST",
     response = Todo.class
 )
@ApiImplicitParams(
     {
          @ApiImplicitParam(
               name = "body",
               dataType = "Todo",
               required = true,
               paramType = "body",
               value = "Todo"
          )
     }
)
@ApiResponses(
          value = {
                  @com.wordnik.swagger.annotations.ApiResponse(code = 400, message = "Json Processing Exception")
          }
)
public static Result createTodo() {
     Form form = Form.form(models.Todo.class).bindFromRequest(); 
     if (form.hasErrors()) {
         return badRequest(form.errorsAsJson());
     }
     else {
          models.Todo todo = form.get();
          todo.user = SecurityController.getUser();
          todo.save();
          return ok(toJson(todo));
     }
}

How to learn as student even you are working

Work on idea any thing for one week 1 hr for day
write the detail on github.com, code also
work on android app screen 1

bike rental in bangalore google map

https://www.google.co.in/search?espv=2&biw=1335&bih=717&q=bike+rental+in+bangalore&npsic=0&rflfq=1&rlha=0&tbm=lcl&sa=X&ved=0CDQQtgNqFQoTCK-B-NejvMgCFY6OjgodRM0O9A

spot book mark search idea

we have spot search in mac.
Why don't we do the same things for book mark search

ideas of the day 2015-10-12

photo processing, uploading schedule on automating

Some business have photographer , they take photos
but they need to rename, processed it, upload it.
It need time
If we can automate, can see the status of progress.

Schedule of photo processing, uploading

Android Google Map integration practice

Reference: https://developers.google.com/maps/documentation/android-api/

Copy file from one ec2 instance to other

suppose you want to transfer file from the second instance(ec2-2) to first instance(ec2-1), the command should be run in ec2-1 is:

scp -i /Path-To-Key-File-for-ec2-2/key.pem ec2-user@Elastic-IP-of-ec2-2:/path/filename your/local-path-on-ec2-1/filename

install github plugin for jenkins

plugin : https://github.com/jenkinsci/jenkins

To install Git Plugin, Navigate to “Jenkins Homepage | Manage Jenkins | Manage Plugins | Available” and search for Git Plugin

Create an IAM Instance Profile for Your Amazon EC2 Instances

reference: http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html

thhgrou aws console

http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html#getting-started-create-ec2-role-console

https://console.aws.amazon.com/iam/.

AWS code pipeline

continuous delivery

when you make changes to your applicaiton
it collects/git pull
builds it
test it
deploy it

in real time


Available Regions:
US East(N. Virginia)

Code reposity support:
Amazon s3
Github

Pythob practice

Type "help", "copyright", "credits" or "license" for more information.

>>> print "Hello Word!"

Hello Word!

>>> if(foo)

  File "", line 1

    if(foo)

          ^

SyntaxError: invalid syntax

>>> f=true

Traceback (most recent call last):

  File "", line 1, in

NameError: name 'true' is not defined

>>> f=1

>>> if f:

...     print "f is 1"

... 

f is 1

>>> if f:

...     print "f is 1"

... else:

...     print "i don't know"

... 

f is 1

>>> """

... multi

... line

... comment

... """

'\nmulti\nline\ncomment\n'

>>> d = None

>>> numbers = [1,2,3,4]

>>> len(numbers)

4

>>> numbers[0]

1

>>> person={}

>>> person["name"]="ikbhal"

>>> person.update({

...     "favourites":[42,"food"],

...     "gender":"male"

... })

>>> person

{'gender': 'male', 'favourites': [42, 'food'], 'name': 'ikbhal'}

>>> person[(44.47, -73)]= "coordinates"

>>> person

{'gender': 'male', 'favourites': [42, 'food'], 'name': 'ikbhal', (44.47, -73): 'coordinates'}

>>> p2= {"name":'Ikbhal', 'gender':'Male'}

>>> p2

{'gender': 'Male', 'name': 'Ikbhal'}

>>> p2.get('name','Anonymous')

'Ikbhal'

>>> p2.values()

['Male', 'Ikbhal']

>>> p2.items()

[('gender', 'Male'), ('name', 'Ikbhal')]

>>> is_python=True

>>> is_python=bool("any object")

>>> is_python

True

>>> animals = "Cats " + "Dogs"

>>> animals += "Rabbits"

>>> name = '%(first)s %(last)s' % {

...     'first':'Ikbhal',

...     'last':'Shaik'}

>>> name

'Ikbhal Shaik'

>>> grade = 82

>>> if grade >= 90:

...     print "A or A+"

... elif grade >= 80:

...     print "B"

... else:

...     print "F"

... 

B

>>> class User(object):

...     pass

... 

>>> class Student(object):

...     def __init__(self, name='Anyonmous')

  File "", line 2

    def __init__(self, name='Anyonmous')

                                       ^

SyntaxError: invalid syntax

Install git on ubuntu

sudo apt-get install git

Amazon ec2 instance accessing from your computer

Create Free tier Amazon ec2 instance
Ubuntu Server 14.04 LTS (HVM), SSD Volume Type - ami-5189a661
Ubuntu Server 14.04 LTS (HVM), EBS General Purpose (SSD) Volume Type. Support available from Canonical (http://www.ubuntu.com/cloud/services).

cd /Users/ikbhal/Documents/work/deployment/;
chmod 400 my-key-pair.pem

ssh -i /Users/ikbhal/Documents/work/deployment/free_tier_ aws_ec2/play-zelo-key-pair.pem ubuntu@ec2-198-51-100-1.compute-1.amazonaws.com

to copy files from local computer to ec2 instance

scp -i my-key-pair.pem SampleFile.txt ubuntu@ec2-198-51-100-1.compute-1.amazonaws.com:~

Play framework deployment on Amazon AWS ec2

refrence : http://www.javacirecep.com/cloud/deploy-play-scala-applications-on-amazon-ec2/

Linux flavour: Amazon Linux AMI

sudo yum remove java-1.7.0-openjdk
sudo yum install java-1.8.0

ec2 type choice info

https://aws.amazon.com/ec2/pricing/

Amazon ec2 container service

Amazon ec2 container service  free of charge
Only charge for ec2
It provides all the container functionalities
manage more containers

redis rails api session

http://billpatrianakos.me/blog/2013/10/14/api-sessions-with-redis-in-rails/

Backend Migration strategy

Full migration

Incremental api group or group wise migration.

Using Redis for Play Framework user fetching based on token

if token exist in the request
check in the redis, if exist use it
else retreive from Monogo and update redis

Different ways to pass secret to play framework application

1)Command line arguments or argument options
2)use environment variable
3)specify production configuration file applicaiton-prod.conf

application.conf.

One way of configuring the application secret on a production server is to pass it as a system property to your start script. For example:

/path/to/yourapp/bin/yourapp -Dplay.crypto.secret="QCY?tAnfk?aZ?iwrNwnxIlR6CTf:G3gf:90Latabg@5241AB`R5W:1uDFN];Ik@n"

This approach is very simple, and we will use this approach in the Play documentation on running your app in production mode as a reminder that the application secret needs to be set. In some environments however, placing secrets in command line arguments is not considered good practice. There are two ways to address this.

Environment variables

The first is to place the application secret in an environment variable. In this case, we recommend you place the following configuration in your application.conf file:

play.crypto.secret="changeme"
play.crypto.secret=${?APPLICATION_SECRET}

The second line in that configuration sets the secret to come from an environment variable called APPLICATION_SECRET if such an environment variable is set, otherwise, it leaves the secret unchanged from the previous line.

This approach works particularly well for cloud based deployment scenarios, where the normal practice is to set passwords and other secrets via environment variables that can be configured through the API for that cloud provider.

Production configuration file

Another approach is to create a production.conf file that lives on the server, and includes application.conf, but also overrides any sensitive configuration, such as the application secret and passwords.

For example:

include "application"

play.crypto.secret="QCY?tAnfk?aZ?iwrNwnxIlR6CTf:G3gf:90Latabg@5241AB`R5W:1uDFN];Ik@n"

Then you can start Play with:

/path/to/yourapp/bin/yourapp -Dconfig.file=/path/to/production.conf