Skip to content

Commit d0438e4

Browse files
markendrmarkendr
committed
crm# 241220-003247 rims project sync by coreid
1 parent 9c101b8 commit d0438e4

4 files changed

Lines changed: 48 additions & 32 deletions

File tree

‎Dockerfile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
# ----------------------------------------------------------------------
22
# BUILD CLOWDER DIST
33
# ----------------------------------------------------------------------
4-
# FROM java:jdk-alpine as clowder-build
5-
FROM openjdk:8-jdk as clowder-build
4+
FROM openjdk:8-jdk-bullseye as clowder-build
65

7-
ARG BRANCH
8-
ARG VERSION
9-
ARG BUILDNUMBER
10-
ARG GITSHA1
6+
ARG BRANCH="unknown"
7+
ARG VERSION="unknown"
8+
ARG BUILDNUMBER="unknown"
9+
ARG GITSHA1="unknown"
1110

11+
# copy these cache folders from $HOME to speed up sbt build during debug
12+
# COPY .sbt /root/.sbt
13+
# COPY .ivy2 /root/.ivy2
1214
WORKDIR /src
1315

1416
# install clowder libraries (hopefully cached)
@@ -31,7 +33,7 @@ RUN rm -rf target/universal/clowder-*.zip clowder clowder-* \
3133
&& ./sbt dist \
3234
&& unzip -q target/universal/clowder-*.zip \
3335
&& mv clowder-* clowder \
34-
&& apk add --no-cache zip \
36+
&& apt-get update && apt-get -y install zip \
3537
&& for x in $(find clowder -name \*.jar); do \
3638
zip -d $x org/apache/log4j/net/JMSAppender.class org/apache/log4j/net/SocketServer.class | grep 'deleting:' && echo "fixed $x"; \
3739
done; \
@@ -41,19 +43,17 @@ RUN rm -rf target/universal/clowder-*.zip clowder clowder-* \
4143
# ----------------------------------------------------------------------
4244
# BUILD CLOWDER
4345
# ----------------------------------------------------------------------
44-
#FROM java:jre-alpine
45-
FROM openjdk:8-jdk
46+
FROM openjdk:8-jre-bullseye as clowder-runtime
4647

4748
# add bash
48-
RUN apt-get update
49-
RUN apt-get install -y bash curl bind9
49+
RUN apt-get update && apt-get install -y bash curl bind9
5050
#RUN apk add --no-cache bash curl
5151

5252
# environemnt variables
53-
ARG BRANCH
54-
ARG VERSION
55-
ARG BUILDNUMBER
56-
ARG GITSHA1
53+
ARG BRANCH="unknown"
54+
ARG VERSION="unknown"
55+
ARG BUILDNUMBER="unknown"
56+
ARG GITSHA1="unknown"
5757
ENV BRANCH=${BRANCH} \
5858
VERSION=${VERSION} \
5959
BUILDNUMBER=${BUILDNUMBER} \
@@ -76,7 +76,7 @@ COPY docker/custom.conf docker/play.plugins /home/clowder/custom/
7676
# add letsecrypt to
7777
# download https://letsencrypt.org/certs/letsencryptauthorityx1.pem
7878
RUN wget https://letsencrypt.org/certs/letsencryptauthorityx1.pem -O /tmp/letsencryptauthorityx1.pem
79-
RUN keytool -import -alias letsecrypt -file /tmp/letsencryptauthorityx1.pem -keystore ${JAVA_HOME}/jre/lib/security/cacerts -storepass changeit
79+
RUN keytool -import -alias letsecrypt -file /tmp/letsencryptauthorityx1.pem -keystore ${JAVA_HOME}/lib/security/cacerts -storepass changeit -noprompt
8080

8181
# Containers should NOT run as root as a good practice
8282
# numeric id to be compatible with openshift, will run as random userid:0

‎app/services/PPMSSyncService.scala‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class PPMSSyncService (application: Application) extends Plugin {
3535
var ppmsUrl: String = ""
3636
var ppmsPumaApiKey: String = ""
3737
var ppmsApi2Key: String = ""
38+
var ppmsCoreids: List[String] = List.empty
3839
var ppmsGetProjectAction: String = ""
3940
var ppmsGetProjectMemberAction: String = ""
4041
var ppmsGetUserAction: String = ""
@@ -55,7 +56,7 @@ class PPMSSyncService (application: Application) extends Plugin {
5556

5657
override def onStart() {
5758
Logger.info("Starting ppms sync plugin")
58-
Logger.info("There are total of:" + spaces.list().length + " spaces")
59+
Logger.info("There are total of: " + spaces.list().length + " spaces")
5960
/*make sure username password is disabled, otherwise turn this off. reason: cannot create username password */
6061
// if ( play.Play.application().configuration().getBoolean("enableUsernamePassword") ) {
6162
// Logger.info("Make sure to turn off usernamepassword to make this plugin works")
@@ -68,6 +69,7 @@ class PPMSSyncService (application: Application) extends Plugin {
6869
this.ppmsUrl = ppmsUrl
6970
this.ppmsPumaApiKey = play.api.Play.configuration.getString("ppms.pumapikey").getOrElse("")
7071
this.ppmsApi2Key = play.api.Play.configuration.getString("ppms.api2key").getOrElse("")
72+
this.ppmsCoreids = play.api.Play.configuration.getString("ppms.coreids").get.trim.split(",").toList
7173
this.ppmsGetProjectAction = play.api.Play.configuration.getString("ppms.action.getprojects").getOrElse("getprojects")
7274
this.ppmsGetProjectMemberAction = play.api.Play.configuration.getString("ppms.action.getprojectmember").getOrElse("getprojectmember")
7375
this.ppmsGetUserAction = play.api.Play.configuration.getString("ppms.action.getuser").getOrElse("getuser")
@@ -180,7 +182,7 @@ class PPMSSyncService (application: Application) extends Plugin {
180182
val projGroup = (projectInfo \ "ProjectGroup").as[String]
181183
val projDesc = (projectInfo \ "Descr").as[String]
182184
var rawDataStorage = (extraProfile \ ppmsStorageField).as[String]
183-
Logger.info(">>>Syncing project: " + projName + " id=" + projId.toString)
185+
Logger.debug("Syncing project: " + projName + " id=" + projId.toString)
184186
if (rawDataStorage == None || rawDataStorage.trim().isEmpty() || !rawDataStorage.contains("-")) {
185187
Logger.info("Project " + projName + " has no storage defined. Ignore!!!")
186188
return
@@ -197,17 +199,17 @@ class PPMSSyncService (application: Application) extends Plugin {
197199
ProjectGroup:%s
198200
Desc:%s""" format(projId, rawDataStorage, projType, projGroup, projDesc)
199201

200-
Logger.info("Syncing project: name =" + projName + " projectId=" + projId + " rawdata=" + rawDataStorage)
202+
Logger.info("Syncing project: name=" + projName + " id=" + projId + " rawdata=" + rawDataStorage)
201203
// allSpaces: List[ProjectSpace]
202204
// val allSpaces = spaces.list()
203205
// Logger.info("There are total of:" + allSpaces.length + " spaces")
204206
// var spaceList = allSpaces.filter(_space => _space.name == projName)
205207
val allSpaces = spaces.listAccess(0, Set[Permission](Permission.ViewSpace), getFirstAdmin, showAll=true, showPublic=true, onlyTrial=false, showOnlyShared=false)
206-
Logger.info("There are total of:" + allSpaces.length + " spaces")
208+
Logger.info("There are total of: " + allSpaces.length + " spaces")
207209
var spaceInDb: Option[ProjectSpace] = None
208210
allSpaces.foreach{aSpace =>
209211
metadatas.getMetadataByAttachTo(ResourceRef(ResourceRef.space, aSpace.id)).foreach { metadata =>
210-
Logger.info("Space "+ aSpace.name + " metadata: " + metadata.content)
212+
Logger.debug("Space " + aSpace.name + " metadata: " + metadata.content)
211213
if (metadata.content != None && projId == (metadata.content \ "projId").as[Int] ) {
212214
spaceInDb = Some(aSpace)
213215
}
@@ -219,6 +221,10 @@ class PPMSSyncService (application: Application) extends Plugin {
219221
spaceInDb match {
220222
case Some(_aSpace) => {
221223
Logger.info(">>>>>>>>>>>>>Space exists, update it<<<<<<<<<<<<<<<")
224+
if (_aSpace.name != projName) {
225+
Logger.info("Updating space name: \"" + _aSpace.name + "\" -> \"" + projName + "\"")
226+
spaces.update(_aSpace.copy(name = projName))
227+
}
222228
metadatas.getMetadataByAttachTo(ResourceRef(ResourceRef.space, _aSpace.id)).foreach { metadata =>
223229
Logger.info("Space "+ _aSpace.name + " metadata: " + metadata.content)
224230
if (metadata.content != None && rawDataStorage.equals( ((metadata.content \ "projStorage").as[String]) ) ) {
@@ -283,15 +289,19 @@ class PPMSSyncService (application: Application) extends Plugin {
283289
return
284290
}
285291
// get projects
286-
val projectsJsonArr = PPMSUtils.getPPMSProjects(ppmsUrl, ppmsPumaApiKey, ppmsGetProjectAction)
287-
projectsJsonArr.value.foreach { projectInfo =>
288-
Logger.info("Syncing project: " + (projectInfo \ "ProjectName").as[String])
289-
val projId = (projectInfo \ "ProjectRef").as[Int]
290-
if ( projId >= startingProjectId ) {
291-
val projectXtraProfileArr = PPMSUtils.getPPMSExtraProjectProfile(ppmsUrl, ppmsApi2Key, projId, ppmsGetXtraProjectProfileAction)
292-
projectXtraProfileArr.value.foreach(syncProject(projectInfo, _))
292+
var numProjects = 0
293+
this.ppmsCoreids.foreach { ppmsCoreid =>
294+
val projectsJsonArr = PPMSUtils.getPPMSProjects(ppmsUrl, ppmsPumaApiKey, ppmsGetProjectAction, ppmsCoreid)
295+
numProjects += projectsJsonArr.value.size
296+
projectsJsonArr.value.foreach { projectInfo =>
297+
val projId = (projectInfo \ "ProjectRef").as[Int]
298+
if ( projId >= startingProjectId ) {
299+
val projectXtraProfileArr = PPMSUtils.getPPMSExtraProjectProfile(ppmsUrl, ppmsApi2Key, projId, ppmsGetXtraProjectProfileAction)
300+
projectXtraProfileArr.value.foreach(syncProject(projectInfo, _))
301+
}
293302
}
294303
}
304+
Logger.info("Synced " + numProjects + " projects")
295305
} // end syncProjectsFromPPMS
296306

297307

‎app/util/PPMSUtils.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,18 @@ object PPMSUtils {
6060
/**
6161
* getPPMSProjects
6262
*/
63-
def getPPMSProjects(url: String, pumaKey: String, getProjectAction: String): JsArray = {
63+
def getPPMSProjects(url: String, pumaKey: String, getProjectAction: String, coreid: String): JsArray = {
6464
val params = new ArrayList[NameValuePair]()
6565
params.add(new BasicNameValuePair("apikey", pumaKey))
6666
params.add(new BasicNameValuePair("action", getProjectAction))
67+
params.add(new BasicNameValuePair("coreid", coreid))
6768
params.add(new BasicNameValuePair("format", "json"))
6869
params.add(new BasicNameValuePair("active", true.toString))
6970
val ppmsResponse = requestPPMS(url, false, params)
7071
ppmsResponse match {
7172
case Some(resp) => {
7273
val projectsList: JsArray = Json.parse(resp).as[JsArray]
74+
Logger.info("PPMS get projects: got " + projectsList.value.size + " projects for coreid " + coreid)
7375
return projectsList
7476
}
7577
case None => {
@@ -162,4 +164,4 @@ object PPMSUtils {
162164
}
163165

164166

165-
}
167+
}

‎project/Build.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import NativePackagerKeys._
1313
object ApplicationBuild extends Build {
1414

1515
val appName = "clowder"
16-
val version = "1.20.1"
16+
val version = "1.20.5"
1717
val jvm = "1.7"
1818

1919
def appVersion: String = {
@@ -119,7 +119,8 @@ object ApplicationBuild extends Build {
119119
"org.scalatestplus" % "play_2.10" % "1.0.0" % "test",
120120

121121
// iRods filestorage
122-
"org.irods.jargon" % "jargon-core" % "3.3.3-beta1",
122+
// see https://github.com/clowder-framework/clowder/issues/457
123+
// "org.irods.jargon" % "jargon-core" % "3.3.3-beta1",
123124

124125
// jsonp return from /api
125126
"org.julienrf" %% "play-jsonp-filter" % "1.1",
@@ -167,6 +168,9 @@ object ApplicationBuild extends Build {
167168
// custom/custom.conf - to customize application.conf
168169
scriptClasspath += "../custom",
169170

171+
// see https://github.com/clowder-framework/clowder/issues/457
172+
excludeFilter in Compile := "IRODS*.scala",
173+
170174
// same for development mode
171175
unmanagedClasspath in Runtime += baseDirectory.value / "custom",
172176

0 commit comments

Comments
 (0)