From 30b409f891b7603af62e2fe243fdf156ebed6aa9 Mon Sep 17 00:00:00 2001
From: Shane Kilkelly <shane@kilkelly.me>
Date: Wed, 20 Jun 2018 10:01:03 +0100
Subject: [PATCH] Refactor the LinkedFiles/Agent system, and track build_id for
output files
---
.../Features/Compile/ClsiManager.coffee | 4 +-
.../LinkedFiles/LinkedFilesController.coffee | 60 ++----
.../LinkedFiles/LinkedFilesErrors.coffee | 122 +++++++++++
.../LinkedFiles/LinkedFilesHandler.coffee | 53 +++++
.../LinkedFiles/ProjectFileAgent.coffee | 189 ++++++++----------
.../LinkedFiles/ProjectOutputFileAgent.coffee | 171 +++++++++++-----
.../Features/LinkedFiles/UrlAgent.coffee | 80 +++-----
app/coffee/infrastructure/FileWriter.coffee | 5 +-
.../controllers/FileTreeController.coffee | 6 +-
.../acceptance/coffee/LinkedFilesTests.coffee | 4 +-
.../coffee/helpers/MockClsiApi.coffee | 18 +-
11 files changed, 460 insertions(+), 252 deletions(-)
create mode 100644 app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee
create mode 100644 app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee
diff --git a/app/coffee/Features/Compile/ClsiManager.coffee b/app/coffee/Features/Compile/ClsiManager.coffee
index 232b416e6..d1dfe21da 100755
--- a/app/coffee/Features/Compile/ClsiManager.coffee
+++ b/app/coffee/Features/Compile/ClsiManager.coffee
@@ -182,8 +182,8 @@ module.exports = ClsiManager =
return callback(error) if error?
callback(null, projectStateHash, docs)
- getOutputFileStream: (project_id, output_file_path, callback=(err, readStream)->) ->
- url = "#{Settings.apis.clsi.url}/project/#{project_id}/output/#{output_file_path}"
+ getOutputFileStream: (project_id, user_id, build_id, output_file_path, callback=(err, readStream)->) ->
+ url = "#{Settings.apis.clsi.url}/project/#{project_id}/user/#{user_id}/build/#{build_id}/output/#{output_file_path}"
ClsiCookieManager.getCookieJar project_id, (err, jar)->
return callback(err) if err?
options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar }
diff --git a/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
index 00c061778..2d42c2e19 100644
--- a/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
+++ b/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee
@@ -4,8 +4,11 @@ ProjectLocator = require '../Project/ProjectLocator'
Settings = require 'settings-sharelatex'
logger = require 'logger-sharelatex'
_ = require 'underscore'
+LinkedFilesErrors = require './LinkedFilesErrors'
+
module.exports = LinkedFilesController = {
+
Agents: {
url: require('./UrlAgent'),
project_file: require('./ProjectFileAgent'),
@@ -38,16 +41,16 @@ module.exports = LinkedFilesController = {
if !Agent?
return res.sendStatus(400)
- linkedFileData = Agent.sanitizeData(data)
- linkedFileData.provider = provider
-
- if !Agent.canCreate(linkedFileData)
- return res.status(403).send('Cannot create linked file')
+ data.provider = provider
- LinkedFilesController._doImport(
- req, res, next, Agent, project_id, user_id,
- parent_folder_id, name, linkedFileData
- )
+ Agent.createLinkedFile project_id,
+ data,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, newFileId) ->
+ return LinkedFilesErrors.handleError(err, req, res, next) if err?
+ res.json(new_file_id: newFileId)
refreshLinkedFile: (req, res, next) ->
{project_id, file_id} = req.params
@@ -66,37 +69,14 @@ module.exports = LinkedFilesController = {
Agent = LinkedFilesController._getAgent(provider)
if !Agent?
return res.sendStatus(400)
- LinkedFilesController._doImport(
- req, res, next, Agent, project_id, user_id,
- parent_folder_id, name, linkedFileData
- )
- _doImport: (req, res, next, Agent, project_id, user_id, parent_folder_id, name, linkedFileData) ->
- Agent.checkAuth project_id, linkedFileData, user_id, (err, allowed) ->
- return Agent.handleError(err, req, res, next) if err?
- return res.sendStatus(403) if !allowed
- Agent.decorateLinkedFileData linkedFileData, (err, newLinkedFileData) ->
- return Agent.handleError(err) if err?
- linkedFileData = newLinkedFileData
- Agent.writeIncomingFileToDisk project_id,
- linkedFileData,
- user_id,
- (error, fsPath) ->
- if error?
- logger.error(
- {err: error, project_id, name, linkedFileData, parent_folder_id, user_id},
- 'error writing linked file to disk'
- )
- return Agent.handleError(error, req, res, next)
- EditorController.upsertFile project_id,
- parent_folder_id,
- name,
- fsPath,
- linkedFileData,
- "upload",
- user_id,
- (error, file) ->
- return next(error) if error?
- res.json(new_file_id: file._id) # created
+ Agent.refreshLinkedFile project_id,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, newFileId) ->
+ return LinkedFilesErrors.handleError(err, req, res, next) if err?
+ res.json(new_file_id: newFileId)
}
diff --git a/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee b/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee
new file mode 100644
index 000000000..69c66e11e
--- /dev/null
+++ b/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee
@@ -0,0 +1,122 @@
+UrlFetchFailedError = (message) ->
+ error = new Error(message)
+ error.name = 'UrlFetchFailedError'
+ error.__proto__ = UrlFetchFailedError.prototype
+ return error
+UrlFetchFailedError.prototype.__proto__ = Error.prototype
+
+
+InvalidUrlError = (message) ->
+ error = new Error(message)
+ error.name = 'InvalidUrlError'
+ error.__proto__ = InvalidUrlError.prototype
+ return error
+InvalidUrlError.prototype.__proto__ = Error.prototype
+
+
+OutputFileFetchFailedError = (message) ->
+ error = new Error(message)
+ error.name = 'OutputFileFetchFailedError'
+ error.__proto__ = OutputFileFetchFailedError.prototype
+ return error
+OutputFileFetchFailedError.prototype.__proto__ = Error.prototype
+
+
+AccessDeniedError = (message) ->
+ error = new Error(message)
+ error.name = 'AccessDenied'
+ error.__proto__ = AccessDeniedError.prototype
+ return error
+AccessDeniedError.prototype.__proto__ = Error.prototype
+
+
+BadEntityTypeError = (message) ->
+ error = new Error(message)
+ error.name = 'BadEntityType'
+ error.__proto__ = BadEntityTypeError.prototype
+ return error
+BadEntityTypeError.prototype.__proto__ = Error.prototype
+
+
+BadDataError = (message) ->
+ error = new Error(message)
+ error.name = 'BadData'
+ error.__proto__ = BadDataError.prototype
+ return error
+BadDataError.prototype.__proto__ = Error.prototype
+
+
+ProjectNotFoundError = (message) ->
+ error = new Error(message)
+ error.name = 'ProjectNotFound'
+ error.__proto__ = ProjectNotFoundError.prototype
+ return error
+ProjectNotFoundError.prototype.__proto__ = Error.prototype
+
+
+V1ProjectNotFoundError = (message) ->
+ error = new Error(message)
+ error.name = 'V1ProjectNotFound'
+ error.__proto__ = V1ProjectNotFoundError.prototype
+ return error
+V1ProjectNotFoundError.prototype.__proto__ = Error.prototype
+
+
+SourceFileNotFoundError = (message) ->
+ error = new Error(message)
+ error.name = 'SourceFileNotFound'
+ error.__proto__ = SourceFileNotFoundError.prototype
+ return error
+SourceFileNotFoundError.prototype.__proto__ = Error.prototype
+
+
+module.exports = {
+
+ UrlFetchFailedError,
+ InvalidUrlError,
+ OutputFileFetchFailedError,
+ AccessDeniedError,
+ BadEntityTypeError,
+ BadDataError,
+ ProjectNotFoundError,
+ V1ProjectNotFoundError,
+ SourceFileNotFoundError,
+
+ handleError: (error, req, res, next) ->
+ if error instanceof BadDataError
+ res.status(400).send("The submitted data is not valid")
+
+ else if error instanceof AccessDeniedError
+ res.status(403).send("You do not have access to this project")
+
+ else if error instanceof BadDataError
+ res.status(400).send("The submitted data is not valid")
+
+ else if error instanceof BadEntityTypeError
+ res.status(400).send("The file is the wrong type")
+
+ else if error instanceof SourceFileNotFoundError
+ res.status(404).send("Source file not found")
+
+ else if error instanceof ProjectNotFoundError
+ res.status(404).send("Project not found")
+
+ else if error instanceof V1ProjectNotFoundError
+ res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file")
+
+ else if error instanceof OutputFileFetchFailedError
+ res.status(404).send("Could not get output file")
+
+ else if error instanceof UrlFetchFailedError
+ res.status(422).send(
+ "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
+ )
+
+ else if error instanceof InvalidUrlError
+ res.status(422).send(
+ "Your URL is not valid. Please check it and try again."
+ )
+
+ else
+ next(error)
+}
diff --git a/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee b/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee
new file mode 100644
index 000000000..4eb339103
--- /dev/null
+++ b/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee
@@ -0,0 +1,53 @@
+LinkedFilesErrors = require './LinkedFilesErrors'
+FileWriter = require '../../infrastructure/FileWriter'
+EditorController = require '../Editor/EditorController'
+_ = require 'underscore'
+
+
+module.exports = LinkedFilesHandler =
+
+ importFromStream: (
+ project_id,
+ readStream,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ callback=(err, file)->
+ ) ->
+ callback = _.once(callback)
+ FileWriter.writeStreamToDisk project_id, readStream, (err, fsPath) ->
+ return callback(err) if err?
+ EditorController.upsertFile project_id,
+ parent_folder_id,
+ name,
+ fsPath,
+ linkedFileData,
+ "upload",
+ user_id,
+ (err, file) =>
+ return callback(err) if err?
+ callback(null, file)
+
+ importContent: (
+ project_id,
+ content,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ callback=(err, file)->
+ ) ->
+ callback = _.once(callback)
+ FileWriter.writeContentToDisk project_id, content, (err, fsPath) ->
+ return callback(err) if err?
+ EditorController.upsertFile project_id,
+ parent_folder_id,
+ name,
+ fsPath,
+ linkedFileData,
+ "upload",
+ user_id,
+ (err, file) =>
+ return callback(err) if err?
+ callback(null, file)
diff --git a/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee b/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee
index 1b9f96473..dab91a516 100644
--- a/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee
+++ b/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee
@@ -1,75 +1,97 @@
-FileWriter = require('../../infrastructure/FileWriter')
AuthorizationManager = require('../Authorization/AuthorizationManager')
ProjectLocator = require('../Project/ProjectLocator')
ProjectGetter = require('../Project/ProjectGetter')
Project = require("../../models/Project").Project
DocstoreManager = require('../Docstore/DocstoreManager')
FileStoreHandler = require('../FileStore/FileStoreHandler')
-FileWriter = require('../../infrastructure/FileWriter')
_ = require "underscore"
Settings = require 'settings-sharelatex'
+LinkedFilesHandler = require './LinkedFilesHandler'
+{
+ BadDataError,
+ AccessDeniedError,
+ BadEntityTypeError,
+ SourceFileNotFoundError,
+ ProjectNotFoundError,
+ V1ProjectNotFoundError
+} = require './LinkedFilesErrors'
+module.exports = ProjectFileAgent = {
-AccessDeniedError = (message) ->
- error = new Error(message)
- error.name = 'AccessDenied'
- error.__proto__ = AccessDeniedError.prototype
- return error
-AccessDeniedError.prototype.__proto__ = Error.prototype
-
-
-BadEntityTypeError = (message) ->
- error = new Error(message)
- error.name = 'BadEntityType'
- error.__proto__ = BadEntityTypeError.prototype
- return error
-BadEntityTypeError.prototype.__proto__ = Error.prototype
-
-
-BadDataError = (message) ->
- error = new Error(message)
- error.name = 'BadData'
- error.__proto__ = BadDataError.prototype
- return error
-BadDataError.prototype.__proto__ = Error.prototype
-
-
-ProjectNotFoundError = (message) ->
- error = new Error(message)
- error.name = 'ProjectNotFound'
- error.__proto__ = ProjectNotFoundError.prototype
- return error
-ProjectNotFoundError.prototype.__proto__ = Error.prototype
-
-
-V1ProjectNotFoundError = (message) ->
- error = new Error(message)
- error.name = 'V1ProjectNotFound'
- error.__proto__ = V1ProjectNotFoundError.prototype
- return error
-V1ProjectNotFoundError.prototype.__proto__ = Error.prototype
-
-
-SourceFileNotFoundError = (message) ->
- error = new Error(message)
- error.name = 'SourceFileNotFound'
- error.__proto__ = SourceFileNotFoundError.prototype
- return error
-SourceFileNotFoundError.prototype.__proto__ = Error.prototype
-
+ createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ if !@_canCreate(linkedFileData)
+ return callback(new AccessDeniedError())
+ @_go(project_id, linkedFileData, name, parent_folder_id, user_id, callback)
-module.exports = ProjectFileAgent = {
+ refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ @_go project_id, linkedFileData, name, parent_folder_id, user_id, callback
- V1ProjectNotFoundError
- BadDataError
- ProjectNotFoundError
- V1ProjectNotFoundError
+ _prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) ->
+ @_checkAuth project_id, linkedFileData, user_id, (err, allowed) =>
+ return callback(err) if err?
+ return callback(new AccessDeniedError()) if !allowed
+ @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) =>
+ return callback(err) if err?
+ if !@_validate(newLinkedFileData)
+ return callback(new BadDataError())
+ callback(null, newLinkedFileData)
- _v1ProjectNotFoundMessage: "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file"
+ _go: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ linkedFileData = @_sanitizeData(linkedFileData)
+ @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+ return callback(err) if err?
+ if !@_validate(linkedFileData)
+ return callback(new BadDataError())
+ @_getEntity linkedFileData, user_id, (err, source_project, entity, type) =>
+ return callback(err) if err?
+ if type == 'doc'
+ DocstoreManager.getDoc source_project._id, entity._id, (err, lines) ->
+ return callback(err) if err?
+ LinkedFilesHandler.importContent project_id,
+ lines.join('\n'),
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, file) ->
+ return callback(err) if err?
+ callback(null, file._id) # Created
+ else if type == 'file'
+ FileStoreHandler.getFileStream source_project._id, entity._id, null, (err, fileStream) ->
+ return callback(err) if err?
+ LinkedFilesHandler.importFromStream project_id,
+ fileStream,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, file) ->
+ return callback(err) if err?
+ callback(null, file._id) # Created
+ else
+ callback(new BadEntityTypeError())
+
+ _getEntity:
+ (linkedFileData, current_user_id, callback = (err, entity, type) ->) ->
+ callback = _.once(callback)
+ { source_entity_path } = linkedFileData
+ @_getSourceProject linkedFileData, (err, project) ->
+ return callback(err) if err?
+ source_project_id = project._id
+ ProjectLocator.findElementByPath {
+ project_id: source_project_id,
+ path: source_entity_path
+ }, (err, entity, type) ->
+ if err?
+ if err.toString().match(/^not found.*/)
+ err = new SourceFileNotFoundError()
+ return callback(err)
+ callback(null, project, entity, type)
- sanitizeData: (data) ->
+ _sanitizeData: (data) ->
return _.pick(
data,
+ 'provider',
'source_project_id',
'v1_source_doc_id',
'source_entity_path'
@@ -81,7 +103,7 @@ module.exports = ProjectFileAgent = {
data.source_entity_path?
)
- canCreate: (data) ->
+ _canCreate: (data) ->
# Don't allow creation of linked-files with v1 doc ids
!data.v1_source_doc_id?
@@ -102,13 +124,13 @@ module.exports = ProjectFileAgent = {
else
callback(new BadDataError('neither v1 nor v2 id present'))
- decorateLinkedFileData: (data, callback = (err, newData) ->) ->
+ _decorateLinkedFileData: (data, callback = (err, newData) ->) ->
callback = _.once(callback)
@_getSourceProject data, (err, project) ->
return callback(err) if err?
callback(err, _.extend(data, {source_project_display_name: project.name}))
- checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
+ _checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
callback = _.once(callback)
if !ProjectFileAgent._validate(data)
return callback(new BadDataError())
@@ -117,53 +139,4 @@ module.exports = ProjectFileAgent = {
AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) ->
return callback(err) if err?
callback(null, canRead)
-
- writeIncomingFileToDisk:
- (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
- callback = _.once(callback)
- if !ProjectFileAgent._validate(data)
- return callback(new BadDataError())
- { source_entity_path } = data
- @_getSourceProject data, (err, project) ->
- return callback(err) if err?
- source_project_id = project._id
- ProjectLocator.findElementByPath {
- project_id: source_project_id,
- path: source_entity_path
- }, (err, entity, type) ->
- if err?
- if err.toString().match(/^not found.*/)
- err = new SourceFileNotFoundError()
- return callback(err)
- ProjectFileAgent._writeEntityToDisk source_project_id, entity._id, type, callback
-
- _writeEntityToDisk: (project_id, entity_id, type, callback=(err, location)->) ->
- callback = _.once(callback)
- if type == 'doc'
- DocstoreManager.getDoc project_id, entity_id, (err, lines) ->
- return callback(err) if err?
- FileWriter.writeLinesToDisk entity_id, lines, callback
- else if type == 'file'
- FileStoreHandler.getFileStream project_id, entity_id, null, (err, fileStream) ->
- return callback(err) if err?
- FileWriter.writeStreamToDisk entity_id, fileStream, callback
- else
- callback(new BadEntityTypeError())
-
- handleError: (error, req, res, next) ->
- if error instanceof AccessDeniedError
- res.status(403).send("You do not have access to this project")
- else if error instanceof BadDataError
- res.status(400).send("The submitted data is not valid")
- else if error instanceof BadEntityTypeError
- res.status(400).send("The file is the wrong type")
- else if error instanceof SourceFileNotFoundError
- res.status(404).send("Source file not found")
- else if error instanceof ProjectNotFoundError
- res.status(404).send("Project not found")
- else if error instanceof V1ProjectNotFoundError
- res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage)
- else
- next(error)
- next()
}
diff --git a/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee
index 0c84d3f7d..aff876f86 100644
--- a/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee
+++ b/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee
@@ -1,86 +1,157 @@
-FileWriter = require('../../infrastructure/FileWriter')
AuthorizationManager = require('../Authorization/AuthorizationManager')
ProjectGetter = require('../Project/ProjectGetter')
-FileWriter = require('../../infrastructure/FileWriter')
Settings = require 'settings-sharelatex'
CompileManager = require '../Compile/CompileManager'
-ClsiCookieManager = require '../Compile/ClsiCookieManager'
ClsiManager = require '../Compile/ClsiManager'
ProjectFileAgent = require './ProjectFileAgent'
_ = require "underscore"
-request = require "request"
+LinkedFilesErrors = require './LinkedFilesErrors'
+LinkedFilesHandler = require './LinkedFilesHandler'
+logger = require 'logger-sharelatex'
-OutputFileFetchFailedError = (message) ->
- error = new Error(message)
- error.name = 'OutputFileFetchFailedError'
- error.__proto__ = OutputFileFetchFailedError.prototype
- return error
-OutputFileFetchFailedError.prototype.__proto__ = Error.prototype
+module.exports = ProjectOutputFileAgent = {
+ _prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) ->
+ @_checkAuth project_id, linkedFileData, user_id, (err, allowed) =>
+ return callback(err) if err?
+ return callback(new LinkedFilesErrors.AccessDeniedError()) if !allowed
+ @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) =>
+ return callback(err) if err?
+ if !@_validate(newLinkedFileData)
+ return callback(new BadDataError())
+ callback(null, newLinkedFileData)
-module.exports = ProjectOutputFileAgent = {
+ createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ if !@_canCreate(linkedFileData)
+ return callback(new LinkedFilesErrors.AccessDeniedError())
+ linkedFileData = @_sanitizeData(linkedFileData)
+ @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+ return callback(err) if err?
+ @_getFileStream linkedFileData, user_id, (err, readStream) =>
+ return callback(err) if err?
+ readStream.on "error", callback
+ readStream.on "response", (response) =>
+ if 200 <= response.statusCode < 300
+ readStream.resume()
+ LinkedFilesHandler.importFromStream project_id,
+ readStream,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, file) ->
+ return callback(err) if err?
+ callback(null, file._id) # Created
+ else
+ err = new LinkedFilesErrors.OutputFileFetchFailedError(
+ "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}"
+ )
+ err.statusCode = response.statusCode
+ callback(err)
+
+ refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) =>
+ return callback(err) if err?
+ @_compileAndGetFileStream linkedFileData, user_id, (err, readStream, new_build_id) =>
+ return callback(err) if err?
+ readStream.on "error", callback
+ readStream.on "response", (response) =>
+ if 200 <= response.statusCode < 300
+ readStream.resume()
+ linkedFileData.build_id = new_build_id
+ LinkedFilesHandler.importFromStream project_id,
+ readStream,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, file) ->
+ return callback(err) if err?
+ callback(null, file._id) # Created
+ else
+ err = new LinkedFilesErrors.OutputFileFetchFailedError(
+ "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}"
+ )
+ err.statusCode = response.statusCode
+ callback(err)
- sanitizeData: (data) ->
+
+ _sanitizeData: (data) ->
return {
+ provider: data.provider,
source_project_id: data.source_project_id,
- source_output_file_path: data.source_output_file_path
+ source_output_file_path: data.source_output_file_path,
+ build_id: data.build_id
}
- canCreate: ProjectFileAgent.canCreate
+ _canCreate: ProjectFileAgent._canCreate
_getSourceProject: ProjectFileAgent._getSourceProject
- decorateLinkedFileData: ProjectFileAgent.decorateLinkedFileData
+ _decorateLinkedFileData: ProjectFileAgent._decorateLinkedFileData
_validate: (data) ->
return (
(data.source_project_id? || data.v1_source_doc_id?) &&
- data.source_output_file_path?
+ data.source_output_file_path? &&
+ data.build_id?
)
- checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
+ _checkAuth: (project_id, data, current_user_id, callback = (err, allowed)->) ->
callback = _.once(callback)
- if !ProjectOutputFileAgent._validate(data)
- return callback(new BadDataError())
+ if !@_validate(data)
+ return callback(new LinkedFilesErrors.BadDataError())
@_getSourceProject data, (err, project) ->
return callback(err) if err?
- AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) ->
- return callback(err) if err?
- callback(null, canRead)
+ AuthorizationManager.canUserReadProject current_user_id,
+ project._id,
+ null,
+ (err, canRead) ->
+ return callback(err) if err?
+ callback(null, canRead)
- writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+ _getFileStream: (linkedFileData, user_id, callback=(err, fileStream)->) ->
callback = _.once(callback)
- if !ProjectOutputFileAgent._validate(data)
- return callback(new BadDataError())
- { source_output_file_path } = data
- @_getSourceProject data, (err, project) ->
+ { source_output_file_path, build_id } = linkedFileData
+ @_getSourceProject linkedFileData, (err, project) ->
return callback(err) if err?
source_project_id = project._id
- CompileManager.compile source_project_id, null, {}, (err) ->
- return callback(err) if err?
- ClsiManager.getOutputFileStream source_project_id, source_output_file_path, (err, readStream) ->
+ ClsiManager.getOutputFileStream source_project_id,
+ user_id,
+ build_id,
+ source_output_file_path,
+ (err, readStream) ->
return callback(err) if err?
readStream.pause()
- readStream.on "error", callback
- readStream.on "response", (response) ->
- if 200 <= response.statusCode < 300
- readStream.resume()
- FileWriter.writeStreamToDisk project_id, readStream, callback
- else
- error = new OutputFileFetchFailedError("Output file fetch failed: #{url}")
- error.statusCode = response.statusCode
- callback(error)
+ callback(null, readStream)
- handleError: (error, req, res, next) ->
- if error instanceof ProjectFileAgent.BadDataError
- res.status(400).send("The submitted data is not valid")
- else if error instanceof OutputFileFetchFailedError
- res.status(404).send("Could not get output file")
- else if error instanceof ProjectFileAgent.ProjectNotFoundError
- res.status(404).send("Project not found")
- else if error instanceof ProjectFileAgent.V1ProjectNotFoundError
- res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage)
- else
- next(error)
+ _compileAndGetFileStream: (linkedFileData, user_id, callback=(err, stream, build_id)->) ->
+ callback = _.once(callback)
+ { source_output_file_path } = linkedFileData
+ @_getSourceProject linkedFileData, (err, project) ->
+ return callback(err) if err?
+ source_project_id = project._id
+ CompileManager.compile source_project_id,
+ user_id,
+ {},
+ (err, status, outputFiles) ->
+ return callback(err) if err?
+ if status != 'success'
+ return callback(new LinkedFilesErrors.OutputFileFetchFailedError())
+ outputFile = _.find(
+ outputFiles,
+ (o) => o.path == source_output_file_path
+ )
+ if !outputFile?
+ return callback(new LinkedFilesErrors.OutputFileFetchFailedError())
+ build_id = outputFile.build
+ ClsiManager.getOutputFileStream source_project_id,
+ user_id,
+ build_id,
+ source_output_file_path,
+ (err, readStream) ->
+ return callback(err) if err?
+ readStream.pause()
+ callback(null, readStream, build_id)
}
diff --git a/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/app/coffee/Features/LinkedFiles/UrlAgent.coffee
index d3748cf8d..b5a6e4902 100644
--- a/app/coffee/Features/LinkedFiles/UrlAgent.coffee
+++ b/app/coffee/Features/LinkedFiles/UrlAgent.coffee
@@ -1,67 +1,53 @@
request = require 'request'
-FileWriter = require('../../infrastructure/FileWriter')
_ = require "underscore"
urlValidator = require 'valid-url'
Settings = require 'settings-sharelatex'
+{ InvalidUrlError, UrlFetchFailedError } = require './LinkedFilesErrors'
+LinkedFilesHandler = require './LinkedFilesHandler'
-UrlFetchFailedError = (message) ->
- error = new Error(message)
- error.name = 'UrlFetchFailedError'
- error.__proto__ = UrlFetchFailedError.prototype
- return error
-UrlFetchFailedError.prototype.__proto__ = Error.prototype
-
-InvalidUrlError = (message) ->
- error = new Error(message)
- error.name = 'InvalidUrlError'
- error.__proto__ = InvalidUrlError.prototype
- return error
-InvalidUrlError.prototype.__proto__ = Error.prototype
module.exports = UrlAgent = {
- UrlFetchFailedError: UrlFetchFailedError
- InvalidUrlError: InvalidUrlError
- sanitizeData: (data) ->
+ createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ linkedFileData = @._sanitizeData(linkedFileData)
+ @_getUrlStream project_id, linkedFileData, user_id, (err, readStream) ->
+ return callback(err) if err?
+ readStream.on "error", callback
+ readStream.on "response", (response) ->
+ if 200 <= response.statusCode < 300
+ readStream.resume()
+ LinkedFilesHandler.importFromStream project_id,
+ readStream,
+ linkedFileData,
+ name,
+ parent_folder_id,
+ user_id,
+ (err, file) ->
+ return callback(err) if err?
+ callback(null, file._id) # Created
+ else
+ error = new UrlFetchFailedError("url fetch failed: #{linkedFileData.url}")
+ error.statusCode = response.statusCode
+ callback(error)
+
+ refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) ->
+ @createLinkedFile project_id, linkedFileData, name, parent_folder_id, user_id, callback
+
+ _sanitizeData: (data) ->
return {
+ provider: data.provider
url: @._prependHttpIfNeeded(data.url)
}
- canCreate: (data) -> true
-
- decorateLinkedFileData: (data, callback = (err, newData) ->) ->
- return callback(null, data)
-
- checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) ->
- callback(null, true)
-
- writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
+ _getUrlStream: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
callback = _.once(callback)
url = data.url
if !urlValidator.isWebUri(url)
return callback(new InvalidUrlError("invalid url: #{url}"))
- url = UrlAgent._wrapWithProxy(url)
+ url = @_wrapWithProxy(url)
readStream = request.get(url)
- readStream.on "error", callback
- readStream.on "response", (response) ->
- if 200 <= response.statusCode < 300
- FileWriter.writeStreamToDisk project_id, readStream, callback
- else
- error = new UrlFetchFailedError("url fetch failed: #{url}")
- error.statusCode = response.statusCode
- callback(error)
-
- handleError: (error, req, res, next) ->
- if error instanceof UrlFetchFailedError
- res.status(422).send(
- "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again."
- )
- else if error instanceof InvalidUrlError
- res.status(422).send(
- "Your URL is not valid. Please check it and try again."
- )
- else
- next(error)
+ readStream.pause()
+ callback(null, readStream)
_prependHttpIfNeeded: (url) ->
if !url.match('://')
diff --git a/app/coffee/infrastructure/FileWriter.coffee b/app/coffee/infrastructure/FileWriter.coffee
index 27b1f1692..4e3727333 100644
--- a/app/coffee/infrastructure/FileWriter.coffee
+++ b/app/coffee/infrastructure/FileWriter.coffee
@@ -15,11 +15,14 @@ module.exports = FileWriter =
callback(null)
writeLinesToDisk: (identifier, lines, callback = (error, fsPath)->) ->
+ FileWriter.writeContentToDisk(identifier, lines.join('\n'), callback)
+
+ writeContentToDisk: (identifier, content, callback = (error, fsPath)->) ->
callback = _.once(callback)
fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}"
FileWriter._ensureDumpFolderExists (error) ->
return callback(error) if error?
- fs.writeFile fsPath, lines.join('\n'), (error) ->
+ fs.writeFile fsPath, content, (error) ->
return callback(error) if error?
callback(null, fsPath)
diff --git a/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
index cd3c35b86..8573b3742 100644
--- a/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
+++ b/public/coffee/ide/file-tree/controllers/FileTreeController.coffee
@@ -218,6 +218,7 @@ define [
projectOutputFiles: null # or []
selectedProjectEntity: null
selectedProjectOutputFile: null
+ buildId: null
name: null
$scope.state.inFlight =
projects: false
@@ -349,6 +350,8 @@ define [
filteredFiles = resp.data.outputFiles.filter (f) ->
f.path.match(/.*\.(pdf|png|jpeg|jpg|gif)/)
$scope.data.projectOutputFiles = filteredFiles
+ $scope.data.buildId = filteredFiles?[0]?.build
+ console.log ">> build_id", $scope.data.buildId
_reset(err: false)
else
$scope.data.projectOutputFiles = null
@@ -368,7 +371,8 @@ define [
provider = 'project_output_file'
payload = {
source_project_id: projectId,
- source_output_file_path: $scope.data.selectedProjectOutputFile
+ source_output_file_path: $scope.data.selectedProjectOutputFile,
+ build_id: $scope.data.buildId
}
else
provider = 'project_file'
diff --git a/test/acceptance/coffee/LinkedFilesTests.coffee b/test/acceptance/coffee/LinkedFilesTests.coffee
index 173aefa15..c892848e0 100644
--- a/test/acceptance/coffee/LinkedFilesTests.coffee
+++ b/test/acceptance/coffee/LinkedFilesTests.coffee
@@ -151,7 +151,7 @@ describe "LinkedFiles", ->
source_entity_path: "/#{@source_doc_name}",
}, (error, response, body) =>
expect(response.statusCode).to.equal 403
- expect(body).to.equal 'Cannot create linked file'
+ expect(body).to.equal 'You do not have access to this project'
done()
describe "with a linked project_file from a v1 project that has not been imported", ->
@@ -380,6 +380,7 @@ describe "LinkedFiles", ->
data:
source_project_id: @project_two_id,
source_output_file_path: "output.pdf",
+ build_id: '1234-abcd'
}, (error, response, body) =>
new_file_id = body.new_file_id
@existing_file_id = new_file_id
@@ -393,6 +394,7 @@ describe "LinkedFiles", ->
source_project_id: @project_two_id,
source_output_file_path: "output.pdf",
source_project_display_name: "output-test-two"
+ build_id: '1234-abcd'
}
expect(firstFile.name).to.equal('test.pdf')
done()
diff --git a/test/acceptance/coffee/helpers/MockClsiApi.coffee b/test/acceptance/coffee/helpers/MockClsiApi.coffee
index 5b01b7a81..60424af2a 100644
--- a/test/acceptance/coffee/helpers/MockClsiApi.coffee
+++ b/test/acceptance/coffee/helpers/MockClsiApi.coffee
@@ -33,12 +33,26 @@ module.exports = MockClsiApi =
app.post "/project/:project_id/compile", (req, res, next) =>
res.json {
- outputFiles: [{path: 'output.pdf'}]
+ compile:
+ status: 'success'
+ outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}]
+ }
+ app.post "/project/:project_id/user/:user_id/compile", (req, res, next) =>
+ res.json {
+ compile:
+ status: 'success'
+ outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}]
}
- app.get "/project/:project_id/output/:output_path", (req, res, next) =>
+ app.get "/project/:project_id/status", (req, res, next) =>
+ res.status(200).send()
+
+ app.get "/project/:project_id/user/:user_id/build/:build_id/output/:output_path", (req, res, next) =>
res.status(200).send("hello")
+ app.all "*", (req, res, next) =>
+ next()
+
app.listen 3013, (error) ->
throw error if error?
.on "error", (error) ->
--
GitLab