stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java [182:438]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return stuff; } @Override protected ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ /** * final UUID version = value.getVersion(); final Field field = value.getField(); final FieldTypeName fieldType = field.getTypeName(); final String fieldValue = field.getValue().toString().toLowerCase(); DynamicComposite composite = new DynamicComposite( ); //we want to sort ascending to descending by version composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); composite.addComponent( field.getName(), STRING_SERIALIZER ); composite.addComponent( fieldValue, STRING_SERIALIZER ); composite.addComponent( fieldType.name() , STRING_SERIALIZER); */ // values are serialized as strings, not sure why, and always lower cased String fieldValueString = fieldEntry.getField().getValue().toString().toLowerCase(); List keys = new ArrayList<>(4); keys.add(fieldEntry.getVersion()); keys.add(fieldEntry.getField().getName()); keys.add(fieldValueString); keys.add(fieldEntry.getField().getTypeName().name()); String comparator = UUID_TYPE_REVERSED; int size = 16+fieldEntry.getField().getName().getBytes().length +fieldEntry.getField().getValue().toString().getBytes().length+ fieldEntry.getField().getTypeName().name().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); for (Object key : keys) { if(key.equals(fieldEntry.getVersion())) { int p = comparator.indexOf("(reversed=true)"); boolean desc = false; if (p >= 0) { comparator = comparator.substring(0, p); desc = true; } byte a = (byte) 85; // this is the byte value for UUIDType in astyanax used in legacy data if (desc) { a = (byte) Character.toUpperCase((char) a); } stuff.putShort((short) ('耀' | a)); }else{ comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); } ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); if (kb == null) { kb = ByteBuffer.allocate(0); } // put a short that indicates how big the buffer is for this item stuff.putShort((short) kb.remaining()); // put the actual item stuff.put(kb.slice()); // put an equality byte ( again not used by part of legacy thrift Astyanax schema) stuff.put((byte) 0); } stuff.flip(); return stuff; } @Override protected ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ){ return serializeKey(applicationId.getUuid(), applicationId.getType(), entityType, fieldType, fieldName, fieldValue); } @Override protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueValueId){ return serializeLogKey(applicationId.getUuid(), applicationId.getType(), uniqueValueId.getUuid(), uniqueValueId.getType()); } @Override protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ /** * final Id entityId = ev.getEntityId(); final UUID entityUuid = entityId.getUuid(); final String entityType = entityId.getType(); CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); builder.addUUID( entityVersion ); builder.addUUID( entityUuid ); builder.addString(entityType ); */ String comparator = "UTF8Type"; List keys = new ArrayList<>(3); keys.add(entityVersion.getEntityVersion()); keys.add(entityVersion.getEntityId().getUuid()); keys.add(entityVersion.getEntityId().getType()); // UUIDs are 16 bytes int size = 16+16+entityVersion.getEntityId().getType().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // we always add comparator to the buffer as well size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); for (Object key : keys) { if(key instanceof UUID){ comparator = "UUIDType"; }else{ comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text } stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); if (kb == null) { kb = ByteBuffer.allocate(0); } // put a short that indicates how big the buffer is for this item stuff.putShort((short) kb.remaining()); // put the actual item stuff.put(kb.slice()); // put an equality byte ( again not used by part of legacy thrift Astyanax schema) stuff.put((byte) 0); } stuff.flip(); return stuff; } @Override protected List deserializeUniqueValueColumn(ByteBuffer bb){ List stuff = new ArrayList<>(); int count = 0; while(bb.hasRemaining()){ // pull of custom comparator (per Astyanax deserialize) int e = CQLUtils.getShortLength(bb); if((e & '耀') == 0) { CQLUtils.getBytes(bb, e); } else { // do nothing } ByteBuffer data = CQLUtils.getWithShortLength(bb); // first two composites are UUIDs, rest are strings if(count == 0) { stuff.add(new UUID(data.getLong(), data.getLong())); }else if(count ==1){ stuff.add(new UUID(data.getLong(), data.getLong())); }else{ stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer count++; } return stuff; } @Override protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ /** * List keys = new ArrayList<>(4); keys.add(fieldEntry.getVersion()); keys.add(fieldEntry.getField().getName()); keys.add(fieldValueString); keys.add(fieldEntry.getField().getTypeName().name()); */ List stuff = new ArrayList<>(); int count = 0; while(bb.hasRemaining()){ // pull of custom comparator (per Astyanax deserialize) int e = CQLUtils.getShortLength(bb); if((e & '耀') == 0) { CQLUtils.getBytes(bb, e); } else { // do nothing } ByteBuffer data = CQLUtils.getWithShortLength(bb); // first composite is a UUID, rest are strings if(count == 0) { stuff.add(new UUID(data.getLong(), data.getLong())); }else{ stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer count++; } return stuff; } @Override public int getImplementationVersion() { - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java [168:427]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return stuff; } @Override protected ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ /** * final UUID version = value.getVersion(); final Field field = value.getField(); final FieldTypeName fieldType = field.getTypeName(); final String fieldValue = field.getValue().toString().toLowerCase(); DynamicComposite composite = new DynamicComposite( ); //we want to sort ascending to descending by version composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); composite.addComponent( field.getName(), STRING_SERIALIZER ); composite.addComponent( fieldValue, STRING_SERIALIZER ); composite.addComponent( fieldType.name() , STRING_SERIALIZER); */ // values are serialized as strings, not sure why, and always lower cased String fieldValueString = fieldEntry.getField().getValue().toString().toLowerCase(); List keys = new ArrayList<>(4); keys.add(fieldEntry.getVersion()); keys.add(fieldEntry.getField().getName()); keys.add(fieldValueString); keys.add(fieldEntry.getField().getTypeName().name()); String comparator = UUID_TYPE_REVERSED; int size = 16+fieldEntry.getField().getName().getBytes().length +fieldEntry.getField().getValue().toString().getBytes().length+ fieldEntry.getField().getTypeName().name().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); for (Object key : keys) { if(key.equals(fieldEntry.getVersion())) { int p = comparator.indexOf("(reversed=true)"); boolean desc = false; if (p >= 0) { comparator = comparator.substring(0, p); desc = true; } byte a = (byte) 85; // this is the byte value for UUIDType in astyanax used in legacy data if (desc) { a = (byte) Character.toUpperCase((char) a); } stuff.putShort((short) ('耀' | a)); }else{ comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); } ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); if (kb == null) { kb = ByteBuffer.allocate(0); } // put a short that indicates how big the buffer is for this item stuff.putShort((short) kb.remaining()); // put the actual item stuff.put(kb.slice()); // put an equality byte ( again not used by part of legacy thrift Astyanax schema) stuff.put((byte) 0); } stuff.flip(); return stuff; } @Override protected ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ){ return serializeKey(applicationId.getUuid(), applicationId.getType(), entityType, fieldType, fieldName, fieldValue); } @Override protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueValueId){ return serializeLogKey(applicationId.getUuid(), applicationId.getType(), uniqueValueId.getUuid(), uniqueValueId.getType()); } @Override protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ /** * final Id entityId = ev.getEntityId(); final UUID entityUuid = entityId.getUuid(); final String entityType = entityId.getType(); CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); builder.addUUID( entityVersion ); builder.addUUID( entityUuid ); builder.addString(entityType ); */ String comparator = "UTF8Type"; List keys = new ArrayList<>(3); keys.add(entityVersion.getEntityVersion()); keys.add(entityVersion.getEntityId().getUuid()); keys.add(entityVersion.getEntityId().getType()); // UUIDs are 16 bytes int size = 16+16+entityVersion.getEntityId().getType().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // we always add comparator to the buffer as well size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); for (Object key : keys) { // custom comparator alias to comparator mappings in CQLUtils.COMPOSITE_TYPE ( more leftover from Asytanax ) // the custom mapping is used for schema creation, but datastax driver does not have the alias concept and // we must work with the actual types if(key instanceof UUID){ comparator = "UUIDType"; }else{ comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text } stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); if (kb == null) { kb = ByteBuffer.allocate(0); } // put a short that indicates how big the buffer is for this item stuff.putShort((short) kb.remaining()); // put the actual item stuff.put(kb.slice()); // put an equality byte ( again not used by part of legacy thrift Astyanax schema) stuff.put((byte) 0); } stuff.flip(); return stuff; } @Override protected List deserializeUniqueValueColumn(ByteBuffer bb){ List stuff = new ArrayList<>(); int count = 0; while(bb.hasRemaining()){ // pull of custom comparator (per Astyanax deserialize) int e = CQLUtils.getShortLength(bb); if((e & '耀') == 0) { CQLUtils.getBytes(bb, e); } else { // do nothing } ByteBuffer data = CQLUtils.getWithShortLength(bb); // first two composites are UUIDs, rest are strings if(count == 0) { stuff.add(new UUID(data.getLong(), data.getLong())); }else if(count ==1){ stuff.add(new UUID(data.getLong(), data.getLong())); }else{ stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer count++; } return stuff; } @Override protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ /** * List keys = new ArrayList<>(4); keys.add(fieldEntry.getVersion()); keys.add(fieldEntry.getField().getName()); keys.add(fieldValueString); keys.add(fieldEntry.getField().getTypeName().name()); */ List stuff = new ArrayList<>(); int count = 0; while(bb.hasRemaining()){ int e = CQLUtils.getShortLength(bb); if((e & '耀') == 0) { CQLUtils.getBytes(bb, e); } else { // do nothing } ByteBuffer data = CQLUtils.getWithShortLength(bb); // first composite is a UUID, rest are strings if(count == 0) { stuff.add(new UUID(data.getLong(), data.getLong())); }else{ stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer count++; } return stuff; } @Override public int getImplementationVersion() { - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -