Merge branch 'master' into release

Gruntfile.coffee

@@ -55,6 +55,8 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-execute'
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
+	grunt.loadNpmTasks "grunt-contrib-coffee"
+
 
 	execute = {}
 	for service in SERVICES
@@ -70,6 +72,18 @@ module.exports = (grunt) ->
 				options:
 					limit: SERVICES.length
 					logConcurrentOutput: true
+		coffee:
+			migrate: 
+				expand: true,
+				flatten: false,
+				cwd: './',
+				src: ['./migrations/*.coffee'],
+				dest: './',
+				ext: '.js'
+				options:
+					bare:true
+
+
 
 		availabletasks:
 			tasks:
@@ -147,11 +161,17 @@ module.exports = (grunt) ->
 	grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
 		Helpers.buildUpstartScripts()
 
+
+	grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
+
+
 	Helpers =
 		installService: (service, callback = (error) ->) ->
 			Helpers.cloneGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
+					return callback(error) if error?
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
 						Helpers.runGruntInstall service, (error) ->
 							return callback(error) if error?
@@ -161,6 +181,8 @@ module.exports = (grunt) ->
 			Helpers.updateGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
+					return callback(error) if error?
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
 						Helpers.runGruntInstall service, (error) ->
 							return callback(error) if error?
@@ -214,6 +236,14 @@ module.exports = (grunt) ->
 			proc.on "close", () ->
 				callback()
 
+		# work around for https://github.com/npm/npm/issues/5400
+		# where binary modules are not built due to bug in npm
+		rebuildNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
+			proc.on "close", () ->
+				callback()
+
 		createDataDirs: (callback = (error) ->) ->
 			DIRS = [
 				"tmp/dumpFolder"

README.md

@@ -26,6 +26,8 @@ ShareLaTeX should run on OS X and Linux. You need:
 * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports.
 * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed.
 
+ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage.
+
 Other repositories
 ------------------
 
@@ -78,6 +80,10 @@ The backend API for managing project tags (folders).
 
 An API for running server-side spelling checking on ShareLaTeX documents.
 
+Dropbox
+-------
+
+Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry!
 
 Contributing
 ------------

migrations/1422460849371-doc-lines.coffee

+
+fs = require("fs")
+mongojs = require("mongojs")
+ObjectId = mongojs.ObjectId
+db = mongojs('sharelatex', ['projects', 'docs'])
+_ = require("lodash")
+async = require("async")
+exec = require("child_process").exec
+
+finished_projects_path = "/tmp/finished-projects"
+all_projects_path = "/tmp/all-projects"
+
+printProgress = ->
+	exec "wc #{finished_projects_path}", (error, results) ->
+		#console.log results
+		setTimeout printProgress, 1000 * 30
+
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		#console.log hasBeenProcessed, project_id
+		callback(null, hasBeenProcessed)
+
+loadProjectIds = (callback)->
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		console.log data.length
+		ids = data.split("\n")
+		console.log ids.length
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
+getProjectIds = (callback)->
+	exists = fs.existsSync all_projects_path
+	if exists 
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
+
+markProjectAsProcessed = (project_id, callback)->
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
+
+getAllDocs = (project_id, callback = (error, docs) ->) ->
+	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
+		return callback(error) if error?
+		if !project?
+			console.error("No such project: #{project_id}")
+			return callback("no such project #{project_id}")
+		findAllDocsInProject project, (error, docs) ->
+			return callback(error) if error?
+			return callback null, docs
+
+findAllDocsInProject = (project, callback = (error, docs) ->) ->
+	callback null, _findAllDocsInFolder project.rootFolder[0]
+
+_findDocInFolder = (folder = {}, doc_id, currentPath) ->
+	for doc, i in folder.docs or []
+		if doc?._id? and doc._id.toString() == doc_id.toString()
+			return {
+				doc: doc
+				mongoPath: "#{currentPath}.docs.#{i}"
+			}
+
+	for childFolder, i in folder.folders or []
+		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
+		return result if result?
+
+	return null
+
+_findAllDocsInFolder = (folder = {}) ->
+	docs = folder.docs or []
+	for childFolder in folder.folders or []
+		docs = docs.concat _findAllDocsInFolder childFolder
+	return docs
+
+insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	update = {}
+	update["_id"] = ObjectId(doc_id)
+	update["lines"] = lines
+	update["project_id"] = ObjectId(project_id)
+	update["rev"] = oldRev
+	db.docs.insert _id: ObjectId(doc_id), callback
+
+saveDocsIntoMongo = (project_id, docs, callback)->
+	jobs = _.map docs, (doc)->
+		(cb)->
+			insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
+	async.series jobs, callback
+
+
+processNext = (project_id, callback)->
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
+		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
+			return callback()
+		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback()
+			saveDocsIntoMongo project_id, docs, ->
+				if err?
+					console.error err, project_id, "could not save docs into mongo"
+					return callback()
+				markProjectAsProcessed project_id, ->
+					callback()
+
+getProjectIds (err, ids)->
+	printProgress()
+	jobs = _.map ids, (id)->
+		return (cb)->
+			processNext(id, cb)
+	async.series jobs, (err)->
+		if err?
+			console.error err, "at end of jobs"
+		else
+			console.log "finished"
+		process.exit()
+	
+exports.up = (next)->
+
+	next()
+
+
+exports.down = (next)->
+	next()

package.json

@@ -4,8 +4,11 @@
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "lodash": "^3.0.0",
+    "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
+    "underscore": "^1.7.0"
   },
   "devDependencies": {
     "grunt": "~0.4.2",
@@ -14,6 +17,7 @@
     "grunt-execute": "~0.1.5",
     "grunt-available-tasks": "~0.4.1",
     "grunt-concurrent": "~0.4.3",
+    "grunt-contrib-coffee": "~0.10.1",
     "semver": "~2.2.1",
     "knox": "~0.8.9"
   }