Skip to content
Snippets Groups Projects
Commit 1978c38c authored by Joscha Schmiedt's avatar Joscha Schmiedt
Browse files

FEAT(gRPC): Add SSS support for gRPC 

Related to issue #362 (#362)
parent 67312d11
No related branches found
No related tags found
1 merge request!121Add server side scripting to gRPC API
Pipeline #62066 passed
Stage: info
Stage: test
Stage: deploy
Hide whitespace changes
Inline Side-by-side
src/main/java/org/caosdb/server/grpc/ServerSideScriptingServiceImpl.java
+ 103
− 67
View file @ 1978c38c
...@@ -21,13 +21,14 @@ ...@@ -21,13 +21,14 @@
package org.caosdb.server.grpc; package org.caosdb.server.grpc;
import com.google.protobuf.Timestamp; import com.google.protobuf.Timestamp;
import com.ibm.icu.text.Collator;
import io.grpc.Status; import io.grpc.Status;
import io.grpc.StatusException; import io.grpc.StatusException;
import io.grpc.stub.StreamObserver; import io.grpc.stub.StreamObserver;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.concurrent.atomic.AtomicLong;
import org.apache.shiro.SecurityUtils; import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authz.AuthorizationException;
import org.apache.shiro.authz.UnauthorizedException; import org.apache.shiro.authz.UnauthorizedException;
import org.apache.shiro.subject.Subject; import org.apache.shiro.subject.Subject;
import org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptRequest; import org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptRequest;
...@@ -41,30 +42,29 @@ import org.caosdb.server.accessControl.OneTimeAuthenticationToken; ...@@ -41,30 +42,29 @@ import org.caosdb.server.accessControl.OneTimeAuthenticationToken;
import org.caosdb.server.accessControl.SessionToken; import org.caosdb.server.accessControl.SessionToken;
import org.caosdb.server.entity.FileProperties; import org.caosdb.server.entity.FileProperties;
import org.caosdb.server.entity.Message; import org.caosdb.server.entity.Message;
import org.caosdb.server.scripting.ScriptingPermissions;
import org.caosdb.server.scripting.ServerSideScriptingCaller; import org.caosdb.server.scripting.ServerSideScriptingCaller;
import org.caosdb.server.utils.ServerMessages; import org.caosdb.server.utils.ServerMessages;
import org.restlet.data.Form;
import org.restlet.data.Parameter;
/** /**
* Main entry point for the server-side scripting service of the server's GRPC * Main entry point for the server-side scripting service of the server's GRPC API.
* API.
* *
* @author Joscha Schmiedt <joscha@schmiedt.dev> * @author Joscha Schmiedt <joscha@schmiedt.dev>
*/ */
public class ServerSideScriptingServiceImpl public class ServerSideScriptingServiceImpl extends ServerSideScriptingServiceImplBase {
extends ServerSideScriptingServiceImplBase {
private ServerSideScriptingCaller caller; private ServerSideScriptingCaller caller;
private static AtomicLong executionId = new AtomicLong(0);
@Override @Override
public void executeServerSideScript( public void executeServerSideScript(
ExecuteServerSideScriptRequest request, ExecuteServerSideScriptRequest request,
io.grpc.stub.StreamObserver<org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptResponse> responseObserver) { io.grpc.stub.StreamObserver<org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptResponse>
responseObserver) {
try { try {
AuthInterceptor.bindSubject(); AuthInterceptor.bindSubject();
ExecuteServerSideScriptResponse response = executeScript(request); ExecuteServerSideScriptResponse response = callScript(request);
responseObserver.onNext(response); responseObserver.onNext(response);
responseObserver.onCompleted(); responseObserver.onCompleted();
...@@ -73,24 +73,27 @@ public class ServerSideScriptingServiceImpl ...@@ -73,24 +73,27 @@ public class ServerSideScriptingServiceImpl
} }
} }
public int callScript(List<String> commandLine, Integer timeoutMs, private long incrementAndGetExecutionId() {
List<FileProperties> files, Object authToken) return executionId.incrementAndGet();
throws Message {
this.caller = new ServerSideScriptingCaller(
commandLine.toArray(new String[commandLine.size()]), timeoutMs, files,
authToken);
return caller.invoke();
} }
/**
* Converts the ExecuteServerSideScriptRequest to a command line argument list.
*
* @param request the request containing the script filename and arguments
* @return the command line argument list
* @throws IllegalArgumentException if the script filename is missing
*/
public static ArrayList<String> request2CommandLine(ExecuteServerSideScriptRequest request) public static ArrayList<String> request2CommandLine(ExecuteServerSideScriptRequest request)
throws Message { throws IllegalArgumentException {
if (request.getScriptFilename().length() == 0) { if (request.getScriptFilename().length() == 0) {
throw ServerMessages.SERVER_SIDE_SCRIPT_MISSING_CALL; throw new IllegalArgumentException("Script filename is missing in the request.");
} }
ArrayList<String> commandLine = new ArrayList<>( ArrayList<String> commandLine =
1 + request.getPositionalArgumentsCount() + request.getNamedArgumentsCount()); new ArrayList<>(
1 + request.getPositionalArgumentsCount() + request.getNamedArgumentsCount());
// first item is the script filename // first item is the script filename
commandLine.add(request.getScriptFilename()); commandLine.add(request.getScriptFilename());
...@@ -114,20 +117,20 @@ public class ServerSideScriptingServiceImpl ...@@ -114,20 +117,20 @@ public class ServerSideScriptingServiceImpl
// TODO: isAnonymous, getUser and generateAuthToken are copy-pasted from // TODO: isAnonymous, getUser and generateAuthToken are copy-pasted from
// elsewhere. Consider refactoring. // elsewhere. Consider refactoring.
///////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////
public boolean isAnonymous() { private static boolean isAnonymous() {
return AuthenticationUtils.isAnonymous(getUser()); return AuthenticationUtils.isAnonymous(getUser());
} }
public Subject getUser() { private static Subject getUser() {
Subject ret = SecurityUtils.getSubject(); Subject ret = SecurityUtils.getSubject();
return ret; return ret;
} }
private String[] constructCallStringArray(ArrayList<String> commandLine) { private static String[] constructCallStringArray(ArrayList<String> commandLine) {
return commandLine.toArray(new String[commandLine.size()]); return commandLine.toArray(new String[commandLine.size()]);
} }
public Object generateAuthToken(String call) { private static Object generateAuthToken(String call) {
String purpose = "SCRIPTING:EXECUTE:" + call; String purpose = "SCRIPTING:EXECUTE:" + call;
Object authtoken = OneTimeAuthenticationToken.generateForPurpose(purpose, getUser()); Object authtoken = OneTimeAuthenticationToken.generateForPurpose(purpose, getUser());
if (authtoken != null || isAnonymous()) { if (authtoken != null || isAnonymous()) {
...@@ -135,53 +138,87 @@ public class ServerSideScriptingServiceImpl ...@@ -135,53 +138,87 @@ public class ServerSideScriptingServiceImpl
} }
return SessionToken.generate(getUser()); return SessionToken.generate(getUser());
} }
private static void checkExecutionPermission(Subject user, String call)
throws AuthorizationException {
user.checkPermission(ScriptingPermissions.PERMISSION_EXECUTION(call));
}
///////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////
private ExecuteServerSideScriptResponse executeScript(ExecuteServerSideScriptRequest request) throws Message { public ExecuteServerSideScriptResponse callScript(ExecuteServerSideScriptRequest request)
throws Message {
long startTimeMillis = System.currentTimeMillis();
Timestamp startTime = Timestamp.newBuilder().setSeconds(startTimeMillis / 1000).build();
checkExecutionPermission(getUser(), request.getScriptFilename());
String[] call = constructCallStringArray(request2CommandLine(request)); String[] call = constructCallStringArray(request2CommandLine(request));
// construct caller and invoke (using an empty file list for now) Integer timeoutMs = request.getTimeoutMs() != 0 ? (int) request.getTimeoutMs() : -1;
Integer timeoutMs = request.getTimeoutMs() != 0 ? (int) request.getTimeoutMs() : null;
if (request.getScriptFilesCount() > 0) {
throw new UnsupportedOperationException(
"Uploading files for scripting via gRPC is not supported yet.");
}
// auth token
Object authToken = request.getAuthToken(); Object authToken = request.getAuthToken();
if (authToken == null) { if (authToken == null) {
authToken = generateAuthToken(request.getScriptFilename()); authToken = generateAuthToken(request.getScriptFilename());
} }
caller = new ServerSideScriptingCaller(call, timeoutMs, new ArrayList<FileProperties>(), authToken);
// request.getCommandLineList(), request.getTimeoutMs(), // invoke script
// request.getFilesList(), request.getAuthToken()); caller =
// caller.invoke(); new ServerSideScriptingCaller(call, timeoutMs, new ArrayList<FileProperties>(), authToken);
int returnCode = caller.invoke();
// construct response
String scriptExecutionId = String.valueOf(incrementAndGetExecutionId());
String scriptFilename = request.getScriptFilename();
ServerSideScriptExecutionResult result =
ServerSideScriptExecutionResult.SERVER_SIDE_SCRIPT_EXECUTION_RESULT_SUCCESS;
// response
String scriptExecutionId = "";
String scriptFilename = "";
ServerSideScriptExecutionResult result = ServerSideScriptExecutionResult.SERVER_SIDE_SCRIPT_EXECUTION_RESULT_GENERAL_FAILURE;
int returnCode = 1;
String stdout = ""; String stdout = "";
try {
stdout = caller.getStdOut();
} catch (final IOException e) {
stdout = "Error retreiving stdout";
result = ServerSideScriptExecutionResult.SERVER_SIDE_SCRIPT_EXECUTION_RESULT_GENERAL_FAILURE;
}
String stderr = ""; String stderr = "";
Timestamp startTime = Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); try {
Timestamp endTime = Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); stderr = caller.getStdErr();
int duration_ms = 0; } catch (final IOException e) {
stderr = "Error retrieving stderr";
final ExecuteServerSideScriptResponse response = ExecuteServerSideScriptResponse.newBuilder() result = ServerSideScriptExecutionResult.SERVER_SIDE_SCRIPT_EXECUTION_RESULT_GENERAL_FAILURE;
.setScriptExecutionId(ServerSideScriptExecutionId.newBuilder() }
.setScriptExecutionId(scriptExecutionId)
.build()) long endTimeMillis = System.currentTimeMillis();
.setCall(scriptFilename) Timestamp endTime = Timestamp.newBuilder().setSeconds(endTimeMillis / 1000).build();
.setResult(result) int duration_ms = (int) (endTimeMillis - startTimeMillis);
.setReturnCode(returnCode)
.setStdout(stdout) final ExecuteServerSideScriptResponse response =
.setStderr(stderr) ExecuteServerSideScriptResponse.newBuilder()
.setStartTime(startTime) .setScriptExecutionId(
.setEndTime(endTime) ServerSideScriptExecutionId.newBuilder()
.setDurationMs(duration_ms) .setScriptExecutionId(scriptExecutionId)
.build(); .build())
.setCall(scriptFilename)
.setResult(result)
.setReturnCode(returnCode)
.setStdout(stdout)
.setStderr(stderr)
.setStartTime(startTime)
.setEndTime(endTime)
.setDurationMs(duration_ms)
.build();
return response; return response;
} }
public static void handleException(StreamObserver<?> responseObserver, public static void handleException(StreamObserver<?> responseObserver, Exception e) {
Exception e) {
String description = e.getMessage(); String description = e.getMessage();
if (description == null || description.isBlank()) { if (description == null || description.isBlank()) {
description = "Unknown Error. Please Report!"; description = "Unknown Error. Please Report!";
...@@ -189,26 +226,25 @@ public class ServerSideScriptingServiceImpl ...@@ -189,26 +226,25 @@ public class ServerSideScriptingServiceImpl
if (e instanceof UnauthorizedException) { if (e instanceof UnauthorizedException) {
Subject subject = SecurityUtils.getSubject(); Subject subject = SecurityUtils.getSubject();
if (AuthenticationUtils.isAnonymous(subject)) { if (AuthenticationUtils.isAnonymous(subject)) {
responseObserver.onError( responseObserver.onError(new StatusException(AuthInterceptor.PLEASE_LOG_IN));
new StatusException(AuthInterceptor.PLEASE_LOG_IN));
return; return;
} else { } else {
responseObserver.onError(new StatusException( responseObserver.onError(
Status.PERMISSION_DENIED.withCause(e).withDescription( new StatusException(
description))); Status.PERMISSION_DENIED.withCause(e).withDescription(description)));
return; return;
} }
} else if (e == ServerMessages.ROLE_DOES_NOT_EXIST || } else if (e == ServerMessages.ROLE_DOES_NOT_EXIST
e == ServerMessages.ACCOUNT_DOES_NOT_EXIST) { || e == ServerMessages.ACCOUNT_DOES_NOT_EXIST) {
responseObserver.onError(new StatusException( responseObserver.onError(
Status.NOT_FOUND.withDescription(description).withCause(e))); new StatusException(Status.NOT_FOUND.withDescription(description).withCause(e)));
return; return;
} }
// TODO: SERVER_SIDE_DOES_NOT_EXIST, SERVER_SIDE_SCRIPT_NOT_EXECUTABLE, // TODO: SERVER_SIDE_DOES_NOT_EXIST, SERVER_SIDE_SCRIPT_NOT_EXECUTABLE,
// SERVER_SIDE_SCRIPT_ERROR, SERVER_SIDE_SCRIPT_SETUP_ERROR, // SERVER_SIDE_SCRIPT_ERROR, SERVER_SIDE_SCRIPT_SETUP_ERROR,
// SERVER_SIDE_SCRIPT_TIMEOUT, SERVER_SIDE_SCRIPT_MISSING_CALL // SERVER_SIDE_SCRIPT_TIMEOUT, SERVER_SIDE_SCRIPT_MISSING_CALL
e.printStackTrace(); e.printStackTrace();
responseObserver.onError(new StatusException( responseObserver.onError(
Status.UNKNOWN.withDescription(description).withCause(e))); new StatusException(Status.UNKNOWN.withDescription(description).withCause(e)));
} }
} }
src/test/java/org/caosdb/server/grpc/ServerSideScriptingGrpcTest.java
+ 218
− 26
View file @ 1978c38c
...@@ -22,38 +22,230 @@ package org.caosdb.server.grpc; ...@@ -22,38 +22,230 @@ package org.caosdb.server.grpc;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.subject.Subject;
import org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptRequest; import org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptRequest;
import org.caosdb.api.scripting.v1alpha1.ExecuteServerSideScriptResponse;
import org.caosdb.api.scripting.v1alpha1.NamedArgument; import org.caosdb.api.scripting.v1alpha1.NamedArgument;
// Python Test Case import org.caosdb.api.scripting.v1alpha1.ServerSideScriptExecutionResult;
// response = run_server_side_script("my_script.py", import org.caosdb.server.CaosDBServer;
// "pos0", import org.caosdb.server.ServerProperties;
// "pos1", import org.caosdb.server.accessControl.AnonymousAuthenticationToken;
// option1="val1", import org.caosdb.server.accessControl.CredentialsValidator;
// option2="val2", import org.caosdb.server.accessControl.Principal;
// files={"-Ofile": "test_file.txt"}) import org.caosdb.server.accessControl.Role;
// assert response.stderr is None import org.caosdb.server.accessControl.UserSources;
// assert response.code == 0 import org.caosdb.server.database.BackendTransaction;
// assert response.call == ('my_script.py ' import org.caosdb.server.database.access.Access;
// '--option1=val1 --option2=val2 --file=.upload_files/test_file.txt ' import org.caosdb.server.database.backend.interfaces.RetrievePasswordValidatorImpl;
// 'pos0 pos1') import org.caosdb.server.database.backend.interfaces.RetrievePermissionRulesImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveRoleImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveUserImpl;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.database.misc.TransactionBenchmark;
import org.caosdb.server.database.proto.ProtoUser;
import org.caosdb.server.entity.Message;
import org.caosdb.server.permissions.PermissionRule;
import org.caosdb.server.scripting.ScriptingPermissions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
public class ServerSideScriptingGrpcTest { public class ServerSideScriptingGrpcTest {
public static class RetrieveRoleMockup implements RetrieveRoleImpl {
public RetrieveRoleMockup(Access a) {}
@Override
public Role retrieve(String role) throws TransactionException {
Role ret = new Role();
ret.name = "anonymous";
ret.description = "bla";
return ret;
}
@Override
public TransactionBenchmark getBenchmark() {
return null;
}
@Override
public void setTransactionBenchmark(TransactionBenchmark b) {}
}
public static class RetrievePermissionRules implements RetrievePermissionRulesImpl {
public RetrievePermissionRules(Access a) {}
@Override
public HashSet<PermissionRule> retrievePermissionRule(String role) throws TransactionException {
HashSet<PermissionRule> result = new HashSet<>();
result.add(
new PermissionRule(
true, false, ScriptingPermissions.PERMISSION_EXECUTION("anonymous_ok")));
return result;
}
@Override
public TransactionBenchmark getBenchmark() {
return null;
}
@Override
public void setTransactionBenchmark(TransactionBenchmark b) {}
}
public static class RetrieveUserMockUp implements RetrieveUserImpl {
public RetrieveUserMockUp(Access a) {}
@Override
public ProtoUser execute(Principal principal) throws TransactionException {
return new ProtoUser();
}
@Override
public TransactionBenchmark getBenchmark() {
return null;
}
@Override
public void setTransactionBenchmark(TransactionBenchmark b) {}
}
public static class RetrievePasswordValidator implements RetrievePasswordValidatorImpl {
public RetrievePasswordValidator(Access a) {}
@Override
public CredentialsValidator<String> execute(String name) throws TransactionException {
return new CredentialsValidator<String>() {
@Override
public boolean isValid(String credential) {
return true;
}
};
}
@Override
public void setTransactionBenchmark(TransactionBenchmark b) {}
@Override
public TransactionBenchmark getBenchmark() {
return null;
}
}
@BeforeAll
public static void setupShiro() throws IOException {
CaosDBServer.initServerProperties();
CaosDBServer.initShiro();
BackendTransaction.setImpl(RetrieveRoleImpl.class, RetrieveRoleMockup.class);
BackendTransaction.setImpl(RetrievePermissionRulesImpl.class, RetrievePermissionRules.class);
BackendTransaction.setImpl(RetrieveUserImpl.class, RetrieveUserMockUp.class);
BackendTransaction.setImpl(
RetrievePasswordValidatorImpl.class, RetrievePasswordValidator.class);
UserSources.getDefaultRealm();
}
@Test @Test
public void testExecuteServerSideScriptRequestToCommandline() { public void testAnonymousWithOutPermission() {
ExecuteServerSideScriptRequest request = ExecuteServerSideScriptRequest.newBuilder() Subject user = SecurityUtils.getSubject();
.setScriptFilename("my_script.py") CaosDBServer.setProperty(ServerProperties.KEY_AUTH_OPTIONAL, "true");
.addPositionalArguments("pos0") user.login(AnonymousAuthenticationToken.getInstance());
.addPositionalArguments("pos1")
.addNamedArguments(NamedArgument.newBuilder().setName("option1").setValue("val1").build()) // TODO: implement this test
.addNamedArguments(NamedArgument.newBuilder().setName("option2").setValue("val2").build())
// .putFiles("-Ofile", "test_file.txt") // Form form = new Form("call=anonymous_no_permission");
.build(); // Representation entity = form.getWebRepresentation();
// Request request = new Request(Method.POST, "../test", entity);
// request.setRootRef(new Reference("bla"));
// request.getAttributes().put("SRID", "asdf1234");
// request.setDate(new Date());
// request.setHostRef("bla");
// resource.init(null, request, new Response(null));
// resource.handle();
// assertEquals(Status.CLIENT_ERROR_FORBIDDEN,
// resource.getResponse().getStatus());
}
@Test
public void testAnonymousWithPermission() {
Subject user = SecurityUtils.getSubject();
CaosDBServer.setProperty(ServerProperties.KEY_AUTH_OPTIONAL, "true");
user.login(AnonymousAuthenticationToken.getInstance());
// TODO: implement this test
// Form form = new Form("call=anonymous_ok");
// Representation entity = form.getWebRepresentation();
// Request request = new Request(Method.POST, "../test", entity);
// request.setRootRef(new Reference("bla"));
// request.getAttributes().put("SRID", "asdf1234");
// request.setDate(new Date());
// request.setHostRef("bla");
// resource.init(null, request, new Response(null));
// resource.handle();
// assertEquals(Status.SUCCESS_OK, resource.getResponse().getStatus());
}
@Test
public void testExecuteServerSideScriptRequestToCommandline() throws Message {
ExecuteServerSideScriptRequest request =
ExecuteServerSideScriptRequest.newBuilder()
.setScriptFilename("my_script.py")
.addPositionalArguments("pos0")
.addPositionalArguments("pos1")
.addNamedArguments(
NamedArgument.newBuilder().setName("option1").setValue("val1").build())
.addNamedArguments(
NamedArgument.newBuilder().setName("option2").setValue("val2").build())
// .putFiles("-Ofile", "test_file.txt")
.build();
ArrayList<String> commandline = ServerSideScriptingServiceImpl.request2CommandLine(request); ArrayList<String> commandline = ServerSideScriptingServiceImpl.request2CommandLine(request);
// assertEquals(commandline,
// "my_script.py --option1=val1 --option2=val2 --file=.upload_files/test_file.txt pos0 pos1"); assertEquals(commandline.get(0), "my_script.py");
// } assertEquals(commandline.get(1), "--option1=val1");
assertEquals(commandline.get(2), "--option2=val2");
assertEquals(commandline.get(3), "pos0");
assertEquals(commandline.get(4), "pos1");
}
@Test
public void testCallScript() throws Message {
// TODO: Implement this test
// ServerSideScriptingServiceImpl service = new ServerSideScriptingServiceImpl();
// ExecuteServerSideScriptRequest request =
// ExecuteServerSideScriptRequest.newBuilder()
// .setScriptFilename("my_script.py")
// .addPositionalArguments("pos0")
// .addPositionalArguments("pos1")
// .addNamedArguments(
// NamedArgument.newBuilder().setName("option1").setValue("val1").build())
// .addNamedArguments(
// NamedArgument.newBuilder().setName("option2").setValue("val2").build())
// .setTimeoutMs(1000)
// .build();
// ExecuteServerSideScriptResponse response = service.callScript(request);
// assertEquals(response.getCall(), "my_script.py");
// assertEquals(
// response.getResult(),
// ServerSideScriptExecutionResult.SERVER_SIDE_SCRIPT_EXECUTION_RESULT_SUCCESS);
// assertEquals(response.getStdout(), ""); // Assuming the script does not produce any output
// assertEquals(response.getStderr(), ""); // Assuming the script does not produce any error output
// assertEquals(response.getReturnCode(), 0); // Assuming the script returns 0
// assertEquals(response.getDurationMs() > 0, true); // Check if duration is greater than 0
}
} }
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment