Skip to content

Commit 905e5bc

Browse files
kurtisvgkurtisvg
authored
Update DLP de-identification samples. (GoogleCloudPlatform#1927)
* Cleanup deidentify with masking. * Cleanup Deidentify With FPE snippet. * Cleanup Reidentify samples. * Update DeIdentiyWithDateShift. * Update region tags and run formatter. * Refactor InfoType stream into a single builder.
1 parent 7507a9b commit 905e5bc

10 files changed

Lines changed: 565 additions & 908 deletions

File tree

‎dlp/src/main/java/com/example/dlp/DeIdentification.java‎

Lines changed: 0 additions & 737 deletions
This file was deleted.

‎dlp/src/main/java/com/example/dlp/Triggers.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ public static void main(String[] args) throws Exception {
261261
cmd = parser.parse(commandLineOptions, args);
262262
} catch (ParseException e) {
263263
System.out.println(e.getMessage());
264-
formatter.printHelp(DeIdentification.class.getName(), commandLineOptions);
264+
formatter.printHelp(Triggers.class.getName(), commandLineOptions);
265265
System.exit(1);
266266
return;
267267
}

‎dlp/src/main/java/dlp/snippets/DeIdentifyWithDateShift.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package dlp.snippets;
2+
3+
// [START dlp_deidentify_date_shift]
4+
5+
import com.google.cloud.dlp.v2.DlpServiceClient;
6+
import com.google.common.base.Splitter;
7+
import com.google.privacy.dlp.v2.ContentItem;
8+
import com.google.privacy.dlp.v2.DateShiftConfig;
9+
import com.google.privacy.dlp.v2.DeidentifyConfig;
10+
import com.google.privacy.dlp.v2.DeidentifyContentRequest;
11+
import com.google.privacy.dlp.v2.DeidentifyContentResponse;
12+
import com.google.privacy.dlp.v2.FieldId;
13+
import com.google.privacy.dlp.v2.FieldTransformation;
14+
import com.google.privacy.dlp.v2.PrimitiveTransformation;
15+
import com.google.privacy.dlp.v2.ProjectName;
16+
import com.google.privacy.dlp.v2.RecordTransformations;
17+
import com.google.privacy.dlp.v2.Table;
18+
import com.google.privacy.dlp.v2.Value;
19+
import com.google.type.Date;
20+
import java.io.BufferedReader;
21+
import java.io.BufferedWriter;
22+
import java.io.IOException;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.nio.file.Paths;
26+
import java.time.LocalDate;
27+
import java.time.format.DateTimeFormatter;
28+
import java.util.Arrays;
29+
import java.util.List;
30+
import java.util.stream.Collectors;
31+
32+
public class DeIdentifyWithDateShift {
33+
34+
public static void deIdentifyWithDateShift() throws IOException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
String projectId = "YOUR_PROJECT_ID";
37+
Path inputCsvFile = Paths.get("path/to/your/input/file.csv");
38+
Path outputCsvFile = Paths.get("path/to/your/output/file.csv");
39+
deIdentifyWithDateShift(projectId, inputCsvFile, outputCsvFile);
40+
}
41+
42+
public static void deIdentifyWithDateShift(
43+
String projectId, Path inputCsvFile, Path outputCsvFile) throws IOException {
44+
// Initialize client that will be used to send requests. This client only needs to be created
45+
// once, and can be reused for multiple requests. After completing all of your requests, call
46+
// the "close" method on the client to safely clean up any remaining background resources.
47+
try (DlpServiceClient dlp = DlpServiceClient.create()) {
48+
// Read the contents of the CSV file into a Table
49+
List<FieldId> headers;
50+
List<Table.Row> rows;
51+
try (BufferedReader input = Files.newBufferedReader(inputCsvFile)) {
52+
// Parse and convert the first line into header names
53+
headers =
54+
Arrays.stream(input.readLine().split(","))
55+
.map(header -> FieldId.newBuilder().setName(header).build())
56+
.collect(Collectors.toList());
57+
// Parse the remainder of the file as Table.Rows
58+
rows =
59+
input.lines().map(DeIdentifyWithDateShift::parseLineAsRow).collect(Collectors.toList());
60+
}
61+
Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build();
62+
ContentItem item = ContentItem.newBuilder().setTable(table).build();
63+
64+
// Set the maximum days to shift dates backwards (lower bound) or forward (upper bound)
65+
DateShiftConfig dateShiftConfig =
66+
DateShiftConfig.newBuilder().setLowerBoundDays(5).setUpperBoundDays(5).build();
67+
PrimitiveTransformation transformation =
68+
PrimitiveTransformation.newBuilder().setDateShiftConfig(dateShiftConfig).build();
69+
// Specify which fields the DateShift should apply too
70+
List<FieldId> dateFields = Arrays.asList(headers.get(1), headers.get(3));
71+
FieldTransformation fieldTransformation =
72+
FieldTransformation.newBuilder()
73+
.addAllFields(dateFields)
74+
.setPrimitiveTransformation(transformation)
75+
.build();
76+
RecordTransformations recordTransformations =
77+
RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build();
78+
// Specify the config for the de-identify request
79+
DeidentifyConfig deidentifyConfig =
80+
DeidentifyConfig.newBuilder().setRecordTransformations(recordTransformations).build();
81+
82+
// Combine configurations into a request for the service.
83+
DeidentifyContentRequest request =
84+
DeidentifyContentRequest.newBuilder()
85+
.setParent(ProjectName.of(projectId).toString())
86+
.setItem(item)
87+
.setDeidentifyConfig(deidentifyConfig)
88+
.build();
89+
90+
// Send the request and receive response from the service
91+
DeidentifyContentResponse response = dlp.deidentifyContent(request);
92+
93+
// Write the results to the target CSV file
94+
try (BufferedWriter writer = Files.newBufferedWriter(outputCsvFile)) {
95+
Table outTable = response.getItem().getTable();
96+
String headerOut =
97+
outTable.getHeadersList().stream()
98+
.map(FieldId::getName)
99+
.collect(Collectors.joining(","));
100+
writer.write(headerOut + "\n");
101+
102+
List<String> rowOutput =
103+
outTable.getRowsList().stream()
104+
.map(row -> joinRow(row.getValuesList()))
105+
.collect(Collectors.toList());
106+
for (String line : rowOutput) {
107+
writer.write(line + "\n");
108+
}
109+
System.out.println("Content written to file: " + outputCsvFile.toString());
110+
}
111+
}
112+
}
113+
114+
// Convert the string from the csv file into com.google.type.Date
115+
public static Date parseAsDate(String s) {
116+
LocalDate date = LocalDate.parse(s, DateTimeFormatter.ofPattern("MM/dd/yyyy"));
117+
return Date.newBuilder()
118+
.setDay(date.getDayOfMonth())
119+
.setMonth(date.getMonthValue())
120+
.setYear(date.getYear())
121+
.build();
122+
}
123+
124+
// Each row is in the format: Name,BirthDate,CreditCardNumber,RegisterDate
125+
public static Table.Row parseLineAsRow(String line) {
126+
List<String> values = Splitter.on(",").splitToList(line);
127+
Value name = Value.newBuilder().setStringValue(values.get(0)).build();
128+
Value birthDate = Value.newBuilder().setDateValue(parseAsDate(values.get(1))).build();
129+
Value creditCardNumber = Value.newBuilder().setStringValue(values.get(2)).build();
130+
Value registerDate = Value.newBuilder().setDateValue(parseAsDate(values.get(3))).build();
131+
return Table.Row.newBuilder()
132+
.addValues(name)
133+
.addValues(birthDate)
134+
.addValues(creditCardNumber)
135+
.addValues(registerDate)
136+
.build();
137+
}
138+
139+
public static String formatDate(Date d) {
140+
return String.format("%s/%s/%s", d.getMonth(), d.getDay(), d.getYear());
141+
}
142+
143+
public static String joinRow(List<Value> values) {
144+
String name = values.get(0).getStringValue();
145+
String birthDate = formatDate(values.get(1).getDateValue());
146+
String creditCardNumber = values.get(2).getStringValue();
147+
String registerDate = formatDate(values.get(3).getDateValue());
148+
return String.join(",", name, birthDate, creditCardNumber, registerDate);
149+
}
150+
}
151+
// [END dlp_deidentify_date_shift]

‎dlp/src/main/java/dlp/snippets/DeIdentifyWithFpe.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package dlp.snippets;
2+
3+
// [START dlp_deidentify_fpe]
4+
5+
import com.google.cloud.dlp.v2.DlpServiceClient;
6+
import com.google.common.io.BaseEncoding;
7+
import com.google.privacy.dlp.v2.ContentItem;
8+
import com.google.privacy.dlp.v2.CryptoKey;
9+
import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig;
10+
import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet;
11+
import com.google.privacy.dlp.v2.DeidentifyConfig;
12+
import com.google.privacy.dlp.v2.DeidentifyContentRequest;
13+
import com.google.privacy.dlp.v2.DeidentifyContentResponse;
14+
import com.google.privacy.dlp.v2.InfoType;
15+
import com.google.privacy.dlp.v2.InfoTypeTransformations;
16+
import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation;
17+
import com.google.privacy.dlp.v2.InspectConfig;
18+
import com.google.privacy.dlp.v2.KmsWrappedCryptoKey;
19+
import com.google.privacy.dlp.v2.PrimitiveTransformation;
20+
import com.google.privacy.dlp.v2.ProjectName;
21+
import com.google.protobuf.ByteString;
22+
import java.io.IOException;
23+
import java.util.Arrays;
24+
25+
public class DeIdentifyWithFpe {
26+
27+
public static void deIdentifyWithFpe() throws IOException {
28+
// TODO(developer): Replace these variables before running the sample.
29+
String projectId = "YOUR_PROJECT_ID";
30+
String textToDeIdentify = "I'm Gary and my email is gary@example.com";
31+
String kmsKeyName =
32+
"projects/YOUR_PROJECT/"
33+
+ "locations/YOUR_KEYRING_REGION/"
34+
+ "keyRings/YOUR_KEYRING_NAME/"
35+
+ "cryptoKeys/YOUR_KEY_NAME";
36+
String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY";
37+
deIdentifyWithFpe(projectId, textToDeIdentify, kmsKeyName, wrappedAesKey);
38+
}
39+
40+
public static void deIdentifyWithFpe(
41+
String projectId, String textToDeIdentify, String kmsKeyName, String wrappedAesKey)
42+
throws IOException {
43+
// Initialize client that will be used to send requests. This client only needs to be created
44+
// once, and can be reused for multiple requests. After completing all of your requests, call
45+
// the "close" method on the client to safely clean up any remaining background resources.
46+
try (DlpServiceClient dlp = DlpServiceClient.create()) {
47+
// Specify what content you want the service to DeIdentify
48+
ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build();
49+
50+
// Specify the type of info the inspection will look for.
51+
// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
52+
InfoType infoType = InfoType.newBuilder().setName("US_SOCIAL_SECURITY_NUMBER").build();
53+
InspectConfig inspectConfig =
54+
InspectConfig.newBuilder().addAllInfoTypes(Arrays.asList(infoType)).build();
55+
56+
// Specify an encrypted AES-256 key and the name of the Cloud KMS key that encrypted it
57+
KmsWrappedCryptoKey kmsWrappedCryptoKey =
58+
KmsWrappedCryptoKey.newBuilder()
59+
.setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedAesKey)))
60+
.setCryptoKeyName(kmsKeyName)
61+
.build();
62+
CryptoKey cryptoKey = CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build();
63+
64+
// Specify how the info from the inspection should be encrypted.
65+
InfoType surrogateInfoType = InfoType.newBuilder().setName("SSN_TOKEN").build();
66+
CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig =
67+
CryptoReplaceFfxFpeConfig.newBuilder()
68+
.setCryptoKey(cryptoKey)
69+
// Set of characters in the input text. For more info, see
70+
// https://cloud.google.com/dlp/docs/reference/rest/v2/organizations.deidentifyTemplates#DeidentifyTemplate.FfxCommonNativeAlphabet
71+
.setCommonAlphabet(FfxCommonNativeAlphabet.NUMERIC)
72+
.setSurrogateInfoType(surrogateInfoType)
73+
.build();
74+
PrimitiveTransformation primitiveTransformation =
75+
PrimitiveTransformation.newBuilder()
76+
.setCryptoReplaceFfxFpeConfig(cryptoReplaceFfxFpeConfig)
77+
.build();
78+
InfoTypeTransformation infoTypeTransformation =
79+
InfoTypeTransformation.newBuilder()
80+
.setPrimitiveTransformation(primitiveTransformation)
81+
.build();
82+
InfoTypeTransformations transformations =
83+
InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build();
84+
85+
DeidentifyConfig deidentifyConfig =
86+
DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build();
87+
88+
// Combine configurations into a request for the service.
89+
DeidentifyContentRequest request =
90+
DeidentifyContentRequest.newBuilder()
91+
.setParent(ProjectName.of(projectId).toString())
92+
.setItem(contentItem)
93+
.setInspectConfig(inspectConfig)
94+
.setDeidentifyConfig(deidentifyConfig)
95+
.build();
96+
97+
// Send the request and receive response from the service
98+
DeidentifyContentResponse response = dlp.deidentifyContent(request);
99+
100+
// Print the results
101+
System.out.println(
102+
"Text after format-preserving encryption: " + response.getItem().getValue());
103+
}
104+
}
105+
}
106+
// [END dlp_deidentify_fpe]

‎dlp/src/main/java/dlp/snippets/DeIdentifyWithMasking.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package dlp.snippets;
2+
3+
// [START dlp_deidentify_masking]
4+
5+
import com.google.cloud.dlp.v2.DlpServiceClient;
6+
import com.google.privacy.dlp.v2.CharacterMaskConfig;
7+
import com.google.privacy.dlp.v2.ContentItem;
8+
import com.google.privacy.dlp.v2.DeidentifyConfig;
9+
import com.google.privacy.dlp.v2.DeidentifyContentRequest;
10+
import com.google.privacy.dlp.v2.DeidentifyContentResponse;
11+
import com.google.privacy.dlp.v2.InfoType;
12+
import com.google.privacy.dlp.v2.InfoTypeTransformations;
13+
import com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation;
14+
import com.google.privacy.dlp.v2.InspectConfig;
15+
import com.google.privacy.dlp.v2.PrimitiveTransformation;
16+
import com.google.privacy.dlp.v2.ProjectName;
17+
import com.google.privacy.dlp.v2.ReplaceWithInfoTypeConfig;
18+
import java.io.IOException;
19+
import java.util.Arrays;
20+
21+
public class DeIdentifyWithMasking {
22+
23+
public static void deIdentifyWithMasking() throws IOException {
24+
// TODO(developer): Replace these variables before running the sample.
25+
String projectId = "your-project-id";
26+
String textToDeIdentify = "My SSN is 372819127";
27+
deIdentifyWithMasking(projectId, textToDeIdentify);
28+
}
29+
30+
public static void deIdentifyWithMasking(String projectId, String textToDeIdentify)
31+
throws IOException {
32+
// Initialize client that will be used to send requests. This client only needs to be created
33+
// once, and can be reused for multiple requests. After completing all of your requests, call
34+
// the "close" method on the client to safely clean up any remaining background resources.
35+
try (DlpServiceClient dlp = DlpServiceClient.create()) {
36+
37+
// Specify what content you want the service to DeIdentify
38+
ContentItem contentItem = ContentItem.newBuilder().setValue(textToDeIdentify).build();
39+
40+
// Specify the type of info the inspection will look for.
41+
// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
42+
InfoType infoType = InfoType.newBuilder().setName("US_SOCIAL_SECURITY_NUMBER").build();
43+
InspectConfig inspectConfig =
44+
InspectConfig.newBuilder().addAllInfoTypes(Arrays.asList(infoType)).build();
45+
46+
// Specify how the info from the inspection should be masked.
47+
CharacterMaskConfig characterMaskConfig =
48+
CharacterMaskConfig.newBuilder()
49+
.setMaskingCharacter("X") // Character to replace the found info with
50+
.setNumberToMask(5) // How many characters should be masked
51+
.build();
52+
PrimitiveTransformation primitiveTransformation =
53+
PrimitiveTransformation.newBuilder()
54+
.setReplaceWithInfoTypeConfig(ReplaceWithInfoTypeConfig.getDefaultInstance())
55+
.build();
56+
InfoTypeTransformation infoTypeTransformation =
57+
InfoTypeTransformation.newBuilder()
58+
.setPrimitiveTransformation(primitiveTransformation)
59+
.build();
60+
InfoTypeTransformations transformations =
61+
InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformation).build();
62+
63+
DeidentifyConfig deidentifyConfig =
64+
DeidentifyConfig.newBuilder().setInfoTypeTransformations(transformations).build();
65+
66+
// Combine configurations into a request for the service.
67+
DeidentifyContentRequest request =
68+
DeidentifyContentRequest.newBuilder()
69+
.setParent(ProjectName.of(projectId).toString())
70+
.setItem(contentItem)
71+
.setInspectConfig(inspectConfig)
72+
.setDeidentifyConfig(deidentifyConfig)
73+
.build();
74+
75+
// Send the request and receive response from the service
76+
DeidentifyContentResponse response = dlp.deidentifyContent(request);
77+
78+
// Print the results
79+
System.out.println("Text after masking: " + response.getItem().getValue());
80+
}
81+
}
82+
}
83+
// [END dlp_deidentify_masking]

‎dlp/src/main/java/dlp/snippets/InspectBigQueryTable.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
package dlp.snippets;
1818

19-
// [START dlp_inspect_gcs]
19+
// [START dlp_inspect_bigquery]
2020

2121
import com.google.cloud.dlp.v2.DlpServiceClient;
2222
import com.google.cloud.pubsub.v1.AckReplyConsumer;
@@ -162,4 +162,4 @@ public static void inspectBigQueryTable(
162162
}
163163
}
164164
}
165-
// [END dlp_inspect_gcs]
165+
// [END dlp_inspect_bigquery]

0 commit comments

Comments
 (0)