Browse Source

Merge branch 'Release_2.1'

master
heck 3 months ago
parent
commit
1e62ff0bc4
6 changed files with 310 additions and 54 deletions
  1. +1
    -1
      DEPENDENCIES
  2. +15
    -3
      src/codegen/gen_java_Message.ysl2
  3. +1
    -1
      test/java/foundation/pEp/jniadapter/test/Makefile.conf
  4. +127
    -49
      test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java
  5. +34
    -0
      test/java/foundation/pEp/jniadapter/test/jni161/Makefile
  6. +132
    -0
      test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java

+ 1
- 1
DEPENDENCIES View File

@ -2,4 +2,4 @@
## Prefer git tags instead of SHA hashes when possible.
libpEpAdapter=Release_2.1.21
pEpEngine=Release_2.1.41
pEpEngine=Release_2.1.42

+ 15
- 3
src/codegen/gen_java_Message.ysl2 View File

@ -33,16 +33,19 @@ tstylesheet {
import java.util.Date;
import java.util.HashMap;
import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
public class «$cname» implements MessageInterface, AutoCloseable, Serializable {
private static final long serialVersionUID = 2119420428331150924L;
private long handle;
private static AtomicLong instanceCount = new AtomicLong(0);
native long init();
native void release(long handle);
public «$cname»() {
handle = init();
instanceCount.getAndIncrement();
}
private native long _«$cname»(
@ -52,6 +55,7 @@ tstylesheet {
public «$cname»(String mime_text) {
byte[] _mime_text = Utils.toUTF8(mime_text);
handle = _«$cname»(_mime_text);
instanceCount.getAndIncrement();
}
private native byte[] _encodeMIME() throws pEpException;
@ -62,11 +66,19 @@ tstylesheet {
private «$cname»(long h) {
handle = h;
instanceCount.getAndIncrement();
}
public final void close() {
release(handle);
handle = 0;
public synchronized final void close() {
if(handle != 0) {
release(handle);
handle = 0;
instanceCount.getAndDecrement();
}
}
public static synchronized AtomicLong getInstanceCount() {
return instanceCount;
}
final protected long getHandle() {


+ 1
- 1
test/java/foundation/pEp/jniadapter/test/Makefile.conf View File

@ -21,7 +21,7 @@ LD_LIB_PATH=.:$(DIST_DIR)
# Java cmd-line options
JAVA_OPT_ASSERTIONS=-enableassertions
JAVA_OPT_XCHECK=-Xcheck:jni
; JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
# JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
JAVA_OPT_CLASSPATH=-cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
JAVA_OPT_LIBARY_PATH=-Djava.library.path=$(LD_LIB_PATH)


+ 127
- 49
test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java View File

@ -1,78 +1,156 @@
package foundation.pEp.jniadapter.test.jni135;
import foundation.pEp.jniadapter.Blob;
import foundation.pEp.jniadapter.Engine;
import foundation.pEp.jniadapter.Message;
import foundation.pEp.jniadapter.decrypt_message_Return;
import foundation.pEp.jniadapter.test.utils.CTXBase;
import foundation.pEp.jniadapter.*;
import foundation.pEp.jniadapter.test.utils.AdapterTestUtils;
import foundation.pEp.jniadapter.test.utils.model.Role;
import foundation.pEp.jniadapter.test.utils.model.TestModel;
import foundation.pEp.jniadapter.test.utils.model.TestNode;
import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity;
import foundation.pEp.pitytest.AbstractTestContext;
import foundation.pEp.pitytest.TestSuite;
import foundation.pEp.pitytest.TestUnit;
import foundation.pEp.pitytest.utils.TestUtils;
import java.lang.ref.WeakReference;
import java.util.Vector;
import static foundation.pEp.pitytest.TestLogger.log;
/*
Test for JNI-135 - Heavy memory consumption - memory leaks
relates to:
* JNI-148 - Mem-mgmt: Defined behaviour of Message.close()
* JNI-160 - Mem-mgmt: Resource Instrumentation for class Message
We are simply encrypting and decrypting in a cycles with 2 own identities alice and bob.
The idea is that you fire up your mem-monitoring tool of choice and convince yourself about the
reality of the amount leaked memory in relation to the number of cycles.
Since JNI-160, this test also prints the result of Message.getInstanceCount(). Please see ticket for more details.
This test suite proofs 3 things:
* If you ignore mem-mgmt, you WILL leak memory
* If you do mem-mgmt right, you DONT leak memory
* If you lost all references to an unreleased message obj, it will be unreleasable forever.
To experiment with this, you can run a single test only and make use of the options:
* repeatCount - how many iterations/cycles
* msgSizeMB - attachement size of one message
* EncFormat - the encryption format
To run a single test only, comment out the last line:
"TestSuite.getDefault().run();"
and run the TestUnit directly using TestUnit.run():
new TestUnit<ctxtype>("bla", new ctxType(), ctx -> {
// code
}).run();
*/
// FOR TEST CONFIG DO NOT TWEAK HERE, THESE ARE DEFAULTS
class Jni135TestContext extends AbstractTestContext {
// Model
public TestModel<pEpTestIdentity, TestNode<pEpTestIdentity>> model = new TestModel(pEpTestIdentity::new, TestNode::new);
// Basic
public Engine engine;
// Identities
public Identity alice;
public Identity bob;
// Test config defaults
public int repeatCount = 2000;
public int msgSizeMB = 1;
public Message.EncFormat encFormat = Message.EncFormat.PEPEncInlineEA;
public Jni135TestContext init() throws Throwable {
engine = new Engine();
// fetch data for the idents
alice = model.getIdent(Role.ALICE).pEpIdent;
bob = model.getIdent(Role.BOB).pEpIdent;
// create keys etc..
alice = engine.myself(alice);
bob = engine.myself(bob);
class Jni135TestContext extends CTXBase {
@Override
public CTXBase init() throws Throwable {
super.init();
return this;
}
}
class TestAlice {
public static void gc() {
// log("gc start");
Object obj = new Object();
WeakReference ref = new WeakReference<Object>(obj);
obj = null;
while (ref.get() != null) {
System.gc();
public static void proofLeakNonLeak(Jni135TestContext ctx, boolean wannaLeak) {
int cycles = 0;
while (cycles < ctx.repeatCount) {
Message msg1Plain = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
Blob bigBlob = AdapterTestUtils.makeNewTestBlob(ctx.msgSizeMB * 1024 * 1024, "atti1", "text/plain");
Vector<Blob> atts = new Vector<Blob>();
atts.add(bigBlob);
msg1Plain.setAttachments(atts);
Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, ctx.encFormat);
decrypt_message_Return decRet = ctx.engine.decrypt_message(msg1Enc, null, 0);
assert decRet != null : "could not decrypt message";
if (!wannaLeak) {
decRet.dst.close();
msg1Enc.close();
msg1Plain.close();
log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount());
assert Message.getInstanceCount().get() == 0 : "Leaking messages";
} else {
log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount());
assert Message.getInstanceCount().get() > 0 : "We should be leaking messages, actually";
}
}
// log("gc end");
}
public static void main(String[] args) throws Exception {
// TestUtils.readKey();
TestSuite.getDefault().setVerbose(true);
TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN);
CTXBase jni135Ctx = new Jni135TestContext();
Engine.setDebugLogEnabled(false);
new TestUnit<CTXBase>("setDir() == getDir() ", new Jni135TestContext(), ctx -> {
ctx.alice = ctx.engine.myself(ctx.alice);
ctx.bob = ctx.engine.myself(ctx.bob);
int cycles = 0;
while (true) {
Message msg1Plain = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
Blob bigBlob = AdapterTestUtils.makeNewTestBlob(10000000, "atti1", "text/plain");
Vector<Blob> atts = new Vector<Blob>();
atts.add(bigBlob);
msg1Plain.setAttachments(atts);
if (false) {
Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, Message.EncFormat.PEP);
decrypt_message_Return decRet = ctx.engine.decrypt_message(msg1Enc, null, 0);
assert decRet != null : "could not decrypt message";
// decRet.dst.close();
// msg1Enc.close();
// gc();
// msg1Plain.close();
}
log("cycles: " + cycles++);
}
}).run();
{
Jni135TestContext ctxDontLeak = new Jni135TestContext();
// Config
ctxDontLeak.repeatCount = 3;
ctxDontLeak.msgSizeMB = 1;
new TestUnit<Jni135TestContext>("Proof leaking messages ", ctxDontLeak, ctx -> {
proofLeakNonLeak(ctx, false);
});
}
{
Jni135TestContext ctxDoLeak = new Jni135TestContext();
// Config
ctxDoLeak.repeatCount = 3;
ctxDoLeak.msgSizeMB = 1;
new TestUnit<Jni135TestContext>("Proof NOT leaking messages ", ctxDoLeak, ctx -> {
proofLeakNonLeak(ctx, true);
});
}
// TestSuite.getDefault().run();
{
Jni135TestContext ctxLostRefs = new Jni135TestContext();
// Config
ctxLostRefs.repeatCount = 3;
ctxLostRefs.msgSizeMB = 1;
new TestUnit<Jni135TestContext>("Lost refs cant be recovered", ctxLostRefs, ctx -> {
try {
proofLeakNonLeak(ctx, false);
assert false : "Lost references to message objects should never be recoverable";
} catch (Throwable t) {
}
// To run one individual test only
})//.run(); // comment this in
;
}
// AND
TestSuite.getDefault().run(); // comment this out
}
}


+ 34
- 0
test/java/foundation/pEp/jniadapter/test/jni161/Makefile View File

@ -0,0 +1,34 @@
include ../../../../../../../Makefile.conf
include ../Makefile.conf
TEST_UNIT_NAME=jni161
JAVA_CLASSES+= \
TestAlice.class
.PHONY: pitytest compile alice test clean
all: alice compile
pitytest:
$(MAKE) -C $(PITYTEST_DIR)
alice: compile clean-pep-home-alice
cd $(JAVA_CWD);pwd;HOME=$(JAVA_PEP_HOME_DIR_ALICE) $(JAVA) $(JAVA_PKG_BASENAME).$(TEST_UNIT_NAME).TestAlice
compile: $(JAVA_CLASSES) pitytest
%.class: %.java
cd $(JAVA_CWD);javac -cp $(CLASSPATH) $(JAVA_PKG_BASEPATH)/$(TEST_UNIT_NAME)/$<
clean:
rm -f $(JAVA_CLASSES)
rm -f *.class
rm -f *.log
rm -Rf .gnupg
rm -Rf .lldb
clean-pep-home: clean-pep-home-alice
clean-pep-home-alice:
rm -rf $(PEP_HOME_DIR_ALICE)/.pEp

+ 132
- 0
test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java View File

@ -0,0 +1,132 @@
package foundation.pEp.jniadapter.test.jni161;
import foundation.pEp.jniadapter.Engine;
import foundation.pEp.jniadapter.Identity;
import foundation.pEp.jniadapter.Message;
import foundation.pEp.jniadapter.test.utils.AdapterTestUtils;
import foundation.pEp.jniadapter.test.utils.model.Role;
import foundation.pEp.jniadapter.test.utils.model.TestModel;
import foundation.pEp.jniadapter.test.utils.model.TestNode;
import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity;
import foundation.pEp.pitytest.AbstractTestContext;
import foundation.pEp.pitytest.TestSuite;
import foundation.pEp.pitytest.TestUnit;
import foundation.pEp.pitytest.utils.TestUtils;
import java.util.Vector;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import static foundation.pEp.pitytest.TestLogger.log;
class Jni161TestContext extends AbstractTestContext {
// Model
public TestModel<pEpTestIdentity, TestNode<pEpTestIdentity>> model = new TestModel(pEpTestIdentity::new, TestNode::new);
// Basic
public Engine engine;
// Identities
public Identity alice;
public Identity bob;
public int blobSizeMB = 1;
public int blobCount = 3;
public Vector<Message> messages = new Vector<>();
public Jni161TestContext init() throws Throwable {
engine = new Engine();
// fetch data for the idents
alice = model.getIdent(Role.ALICE).pEpIdent;
bob = model.getIdent(Role.BOB).pEpIdent;
// create keys etc..
alice = engine.myself(alice);
bob = engine.myself(bob);
return this;
}
}
class TestAlice {
public static void main(String[] args) throws Exception {
TestSuite.getDefault().setVerbose(true);
TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN);
Jni161TestContext ctx161 = new Jni161TestContext();
new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
while (true) {
log("Thread: " + Thread.currentThread().getId() + " - Alloc");
Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
// msg.close();
TestUtils.sleep(10);
}
});
new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
new Thread(() -> {
while (true) {
log("Thread: " + Thread.currentThread().getId() + " - Alloc");
Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
ctx.messages.add(msg);
}
}).start();
new Thread(() -> {
while (true) {
// log("Thread: " + Thread.currentThread().getId() + " - Dealloc");
if(ctx.messages.size() > 0) {
log("Messages: " + ctx.messages.size());
log("InstanceCount: " + Message.getInstanceCount());
ctx.messages.remove(0).close();
}
}
}).start();
}).run();
new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
ExecutorService allocPool = Executors.newFixedThreadPool(10);
ExecutorService deallocPool = Executors.newFixedThreadPool(10);
while (true) {
ThreadPoolExecutor allocExec = (ThreadPoolExecutor) allocPool;
if (allocExec.getActiveCount() < allocExec.getMaximumPoolSize()) {
allocPool.submit(() -> {
log("Thread: " + Thread.currentThread().getId() + " - Alloc");
Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
ctx.messages.add(msg);
log("MessagesADD: " + ctx.messages.size());
});
}
if (ctx.messages.size() > 1) {
ThreadPoolExecutor deallocExec = (ThreadPoolExecutor) deallocPool;
if (deallocExec.getActiveCount() < deallocExec.getMaximumPoolSize()) {
deallocPool.submit(() -> {
log("Thread: " + Thread.currentThread().getId() + " - Dealloc");
log("Messages: " + ctx.messages.size());
log("InstanceCount: " + Message.getInstanceCount());
for (Message msg : ctx.messages) {
msg.close();
}
});
}
}
TestUtils.sleep(10);
}
});
// TestSuite.getDefault().run(); // comment this out
}
}

Loading…
Cancel
Save