1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17 package org.apache.nifi.processors.standard;
18
19 import org.apache.nifi.annotation.behavior.EventDriven;
20 import org.apache.nifi.annotation.behavior.InputRequirement;
21 import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
22 import org.apache.nifi.annotation.behavior.ReadsAttribute;
23 import org.apache.nifi.annotation.behavior.Restricted;
24 import org.apache.nifi.annotation.behavior.Restriction;
25 import org.apache.nifi.annotation.behavior.SupportsBatching;
26 import org.apache.nifi.annotation.documentation.CapabilityDescription;
27 import org.apache.nifi.annotation.documentation.SeeAlso;
28 import org.apache.nifi.annotation.documentation.Tags;
29 import org.apache.nifi.components.PropertyDescriptor;
30 import org.apache.nifi.components.RequiredPermission;
31 import org.apache.nifi.components.ValidationContext;
32 import org.apache.nifi.components.ValidationResult;
33 import org.apache.nifi.components.Validator;
34 import org.apache.nifi.expression.ExpressionLanguageScope;
35 import org.apache.nifi.flowfile.FlowFile;
36 import org.apache.nifi.flowfile.attributes.CoreAttributes;
37 import org.apache.nifi.logging.ComponentLog;
38 import org.apache.nifi.processor.AbstractProcessor;
39 import org.apache.nifi.processor.ProcessContext;
40 import org.apache.nifi.processor.ProcessSession;
41 import org.apache.nifi.processor.ProcessorInitializationContext;
42 import org.apache.nifi.processor.Relationship;
43 import org.apache.nifi.processor.exception.ProcessException;
44 import org.apache.nifi.processor.util.StandardValidators;
45 import org.apache.nifi.util.StopWatch;
46
47 import java.nio.file.Files;
48 import java.nio.file.Path;
49 import java.nio.file.Paths;
50 import java.nio.file.attribute.PosixFileAttributeView;
51 import java.nio.file.attribute.PosixFilePermissions;
52 import java.nio.file.attribute.UserPrincipalLookupService;
53 import java.text.DateFormat;
54 import java.text.SimpleDateFormat;
55 import java.util.Arrays;
56 import java.util.ArrayList;
57 import java.util.Collections;
58 import java.util.Date;
59 import java.util.HashSet;
60 import java.util.List;
61 import java.util.Locale;
62 import java.util.Set;
63 import java.util.concurrent.TimeUnit;
64 import java.util.regex.Matcher;
65 import java.util.regex.Pattern;
66
67 @EventDriven // Processor Behavior Annotations Processor can be scheduled using the Event-Driven scheduling strategy
68 @SupportsBatching // Processor Behavior Annotations - This annotation indicates that it is okay for the framework to batch together multiple ProcessSession commits into a single commit
69 @InputRequirement(Requirement.INPUT_REQUIRED) // Incoming connection is required
70 @Tags({"put", "local", "copy", "archive", "files", "filesystem"}) // Comments in processor
71 @CapabilityDescription("Writes the contents of a FlowFile to the local file system") // Comments in processor show in "Description"
72 @SeeAlso({FetchFile.class, GetFile.class}) // Comments in processor show in "Description"
73 @ReadsAttribute(attribute = "filename", description = "The filename to use when writing the FlowFile to disk.") // Comments in processor show in "Read Attributes"
74 @Restricted(
75 restrictions = {
76 @Restriction(
77 requiredPermission = RequiredPermission.WRITE_FILESYSTEM,
78 explanation = "Provides operator the ability to write to any file that NiFi has access to.")
79 }
80 ) // Comments in processor show in "Restricted"
81 public class PutFile extends AbstractProcessor { // processor should implement AbstractProcessor
82
83 public static final String REPLACE_RESOLUTION = "replace";
84 public static final String IGNORE_RESOLUTION = "ignore";
85 public static final String FAIL_RESOLUTION = "fail";
86
87 public static final String FILE_MODIFY_DATE_ATTRIBUTE = "file.lastModifiedTime";
88 public static final String FILE_MODIFY_DATE_ATTR_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
89
90 public static final Pattern RWX_PATTERN = Pattern.compile("^([r-][w-])([x-])([r-][w-])([x-])([r-][w-])([x-])$");
91 public static final Pattern NUM_PATTERN = Pattern.compile("^[0-7]{3}$");
92
93 // A PropertyDescriptor MUST specify one or more Validators that can be used to ensure that the user-entered value for a property is valid
94 // This is a self design validator
95 private static final Validator PERMISSIONS_VALIDATOR = new Validator() {
96 @Override
97 public ValidationResult validate(String subject, String input, ValidationContext context) { //
98 ValidationResult.Builder vr = new ValidationResult.Builder();
99 if (context.isExpressionLanguagePresent(input)) {
100 return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
101 }
102
103 if (RWX_PATTERN.matcher(input).matches() || NUM_PATTERN.matcher(input).matches()) {
104 return vr.valid(true).build();
105 }
106 return vr.valid(false)
107 .subject(subject)
108 .input(input)
109 .explanation("This must be expressed in rwxr-x--- form or octal triplet form.")
110 .build();
111 }
112 };
113
114 // All the properties will be shown in Configure -> Properties
115 // Define Property Description
116 public static final PropertyDescriptor DIRECTORY = new PropertyDescriptor.Builder()
117 .name("Directory")
118 .description("The directory to which files should be written. You may use expression language such as /aa/bb/${path}")
119 .required(true)
120 .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
121 .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
122 .build();
123 public static final PropertyDescriptor MAX_DESTINATION_FILES = new PropertyDescriptor.Builder()
124 .name("Maximum File Count")
125 .description("Specifies the maximum number of files that can exist in the output directory")
126 .required(false)
127 .addValidator(StandardValidators.INTEGER_VALIDATOR)
128 .build();
129 public static final PropertyDescriptor CONFLICT_RESOLUTION = new PropertyDescriptor.Builder()
130 .name("Conflict Resolution Strategy")
131 .description("Indicates what should happen when a file with the same name already exists in the output directory")
132 .required(true)
133 .defaultValue(FAIL_RESOLUTION)
134 .allowableValues(REPLACE_RESOLUTION, IGNORE_RESOLUTION, FAIL_RESOLUTION)
135 .build();
136 public static final PropertyDescriptor CHANGE_LAST_MODIFIED_TIME = new PropertyDescriptor.Builder()
137 .name("Last Modified Time")
138 .description("Sets the lastModifiedTime on the output file to the value of this attribute. Format must be yyyy-MM-dd'T'HH:mm:ssZ. "
139 + "You may also use expression language such as ${file.lastModifiedTime}.")
140 .required(false)
141 .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
142 .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
143 .build();
144 public static final PropertyDescriptor CHANGE_PERMISSIONS = new PropertyDescriptor.Builder()
145 .name("Permissions")
146 .description("Sets the permissions on the output file to the value of this attribute. Format must be either UNIX rwxrwxrwx with a - in "
147 + "place of denied permissions (e.g. rw-r--r--) or an octal number (e.g. 644). You may also use expression language such as "
148 + "${file.permissions}.")
149 .required(false)
150 .addValidator(PERMISSIONS_VALIDATOR)
151 .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
152 .build();
153 public static final PropertyDescriptor CHANGE_OWNER = new PropertyDescriptor.Builder()
154 .name("Owner")
155 .description("Sets the owner on the output file to the value of this attribute. You may also use expression language such as "
156 + "${file.owner}. Note on many operating systems Nifi must be running as a super-user to have the permissions to set the file owner.")
157 .required(false)
158 .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
159 .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
160 .build();
161 public static final PropertyDescriptor CHANGE_GROUP = new PropertyDescriptor.Builder()
162 .name("Group")
163 .description("Sets the group on the output file to the value of this attribute. You may also use expression language such "
164 + "as ${file.group}.")
165 .required(false)
166 .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
167 .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
168 .build();
169 public static final PropertyDescriptor CREATE_DIRS = new PropertyDescriptor.Builder()
170 .name("Create Missing Directories")
171 .description("If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.")
172 .required(true)
173 .allowableValues("true", "false")
174 .defaultValue("true")
175 .build();
176
177 public static final int MAX_FILE_LOCK_ATTEMPTS = 10;
178
179 // Define relationship
180 public static final Relationship REL_SUCCESS = new Relationship.Builder()
181 .name("success")
182 .description("Files that have been successfully written to the output directory are transferred to this relationship")
183 .build();
184 public static final Relationship REL_FAILURE = new Relationship.Builder()
185 .name("failure")
186 .description("Files that could not be written to the output directory for some reason are transferred to this relationship")
187 .build();
188
189 private List<PropertyDescriptor> properties;
190 private Set<Relationship> relationships;
191
192 // Init the relationships and descriptors
193 @Override
194 protected void init(final ProcessorInitializationContext context) {
195 // relationships
196 final Set<Relationship> procRels = new HashSet<>();
197 procRels.add(REL_SUCCESS);
198 procRels.add(REL_FAILURE);
199 relationships = Collections.unmodifiableSet(procRels); // relationships can't be changed after init
200
201 // descriptors
202 final List<PropertyDescriptor> supDescriptors = new ArrayList<>();
203 supDescriptors.add(DIRECTORY);
204 supDescriptors.add(CONFLICT_RESOLUTION);
205 supDescriptors.add(CREATE_DIRS);
206 supDescriptors.add(MAX_DESTINATION_FILES);
207 supDescriptors.add(CHANGE_LAST_MODIFIED_TIME);
208 supDescriptors.add(CHANGE_PERMISSIONS);
209 supDescriptors.add(CHANGE_OWNER);
210 supDescriptors.add(CHANGE_GROUP);
211 properties = Collections.unmodifiableList(supDescriptors); // properties can't be changed after init
212 }
213
214 @Override
215 public Set<Relationship> getRelationships() {
216 return relationships;
217 }
218
219 @Override
220 protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
221 return properties;
222 }
223
224 @Override
225 // ProcessContext provides a bridge between a processor and the framework. It can provide the information of a processor
226 // ProcessSession can create, destroy, examine, clone, transfer FlowFile, also can edit FlowFile including edit content and attributes
227 public void onTrigger(final ProcessContext context, final ProcessSession session) {
228 FlowFile flowFile = session.get();
229 if (flowFile == null) {
230 return;
231 }
232
233 // used to "Reporting Processor Activity"
234 // StopWatch is used to record the time costed during the period
235 final StopWatch stopWatch = new StopWatch(true);
236 // Processors should log events via the ComponentLog, which is accessible via the InitializationContext or by calling the getLogger method of AbstractProcessor.
237 final ComponentLog logger = getLogger();
238
239 // get the parameters from the configure property
240 final Path configuredRootDirPath = Paths.get(context.getProperty(DIRECTORY).evaluateAttributeExpressions(flowFile).getValue());
241 final String conflictResponse = context.getProperty(CONFLICT_RESOLUTION).getValue();
242 final Integer maxDestinationFiles = context.getProperty(MAX_DESTINATION_FILES).asInteger();
243
244 Path tempDotCopyFile = null;
245 try {
246 final Path rootDirPath = configuredRootDirPath;
247 String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
248 final Path tempCopyFile = rootDirPath.resolve("." + filename);
249 final Path copyFile = rootDirPath.resolve(filename); // return the absolute path
250
251 final String permissions = context.getProperty(CHANGE_PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue(); // evaluateAttributeExpression
252 final String owner = context.getProperty(CHANGE_OWNER).evaluateAttributeExpressions(flowFile).getValue();
253 final String group = context.getProperty(CHANGE_GROUP).evaluateAttributeExpressions(flowFile).getValue();
254 if (!Files.exists(rootDirPath)) {
255 if (context.getProperty(CREATE_DIRS).asBoolean()) {
256 Path existing = rootDirPath;
257 while (!Files.exists(existing)) {
258 existing = existing.getParent();
259 }
260 if (permissions != null && !permissions.trim().isEmpty()) {
261 try {
262 String perms = stringPermissions(permissions, true);
263 if (!perms.isEmpty()) {
264 Files.createDirectories(rootDirPath, PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(perms)));
265 } else {
266 Files.createDirectories(rootDirPath);
267 }
268 } catch (Exception e) {
269 flowFile = session.penalize(flowFile); // Sets a penalty for the given FlowFile which will make it unavailable to be operated on any further during the penalty period.
270 session.transfer(flowFile, REL_FAILURE); // FlowFile is transferred to next stage with Fail relationship
271 logger.error("Could not set create directory with permissions {} because {}", new Object[]{permissions, e});
272 return;
273 }
274 } else {
275 Files.createDirectories(rootDirPath);
276 }
277
278 boolean chOwner = owner != null && !owner.trim().isEmpty();
279 boolean chGroup = group != null && !group.trim().isEmpty();
280 if (chOwner || chGroup) {
281 Path currentPath = rootDirPath;
282 while (!currentPath.equals(existing)) {
283 if (chOwner) {
284 try {
285 UserPrincipalLookupService lookupService = currentPath.getFileSystem().getUserPrincipalLookupService();
286 Files.setOwner(currentPath, lookupService.lookupPrincipalByName(owner));
287 } catch (Exception e) {
288 logger.warn("Could not set directory owner to {} because {}", new Object[]{owner, e});
289 }
290 }
291 if (chGroup) {
292 try {
293 UserPrincipalLookupService lookupService = currentPath.getFileSystem().getUserPrincipalLookupService();
294 PosixFileAttributeView view = Files.getFileAttributeView(currentPath, PosixFileAttributeView.class);
295 view.setGroup(lookupService.lookupPrincipalByGroupName(group));
296 } catch (Exception e) {
297 logger.warn("Could not set file group to {} because {}", new Object[]{group, e});
298 }
299 }
300 currentPath = currentPath.getParent();
301 }
302 }
303 } else {
304 flowFile = session.penalize(flowFile);
305 session.transfer(flowFile, REL_FAILURE);
306 logger.error("Penalizing {} and routing to 'failure' because the output directory {} does not exist and Processor is "
307 + "configured not to create missing directories", new Object[]{flowFile, rootDirPath});
308 return;
309 }
310 }
311
312 final Path dotCopyFile = tempCopyFile;
313 tempDotCopyFile = dotCopyFile;
314 Path finalCopyFile = copyFile;
315
316 final Path finalCopyFileDir = finalCopyFile.getParent();
317 if (Files.exists(finalCopyFileDir) && maxDestinationFiles != null) { // check if too many files already
318 final long numFiles = getFilesNumberInFolder(finalCopyFileDir, filename);
319
320 if (numFiles >= maxDestinationFiles) {
321 flowFile = session.penalize(flowFile);
322 logger.warn("Penalizing {} and routing to 'failure' because the output directory {} has {} files, which exceeds the "
323 + "configured maximum number of files", new Object[]{flowFile, finalCopyFileDir, numFiles});
324 session.transfer(flowFile, REL_FAILURE);
325 return;
326 }
327 }
328
329 if (Files.exists(finalCopyFile)) {
330 switch (conflictResponse) {
331 case REPLACE_RESOLUTION:
332 Files.delete(finalCopyFile);
333 logger.info("Deleted {} as configured in order to replace with the contents of {}", new Object[]{finalCopyFile, flowFile});
334 break;
335 case IGNORE_RESOLUTION:
336 session.transfer(flowFile, REL_SUCCESS);
337 logger.info("Transferring {} to success because file with same name already exists", new Object[]{flowFile});
338 return;
339 case FAIL_RESOLUTION:
340 flowFile = session.penalize(flowFile);
341 logger.warn("Penalizing {} and routing to failure as configured because file with the same name already exists", new Object[]{flowFile});
342 session.transfer(flowFile, REL_FAILURE);
343 return;
344 default:
345 break;
346 }
347 }
348
349 session.exportTo(flowFile, dotCopyFile, false); // Writes the content of the given FlowFile to the given destination path
350
351 final String lastModifiedTime = context.getProperty(CHANGE_LAST_MODIFIED_TIME).evaluateAttributeExpressions(flowFile).getValue();
352 if (lastModifiedTime != null && !lastModifiedTime.trim().isEmpty()) {
353 try {
354 final DateFormat formatter = new SimpleDateFormat(FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
355 final Date fileModifyTime = formatter.parse(lastModifiedTime);
356 dotCopyFile.toFile().setLastModified(fileModifyTime.getTime());
357 } catch (Exception e) {
358 logger.warn("Could not set file lastModifiedTime to {} because {}", new Object[]{lastModifiedTime, e});
359 }
360 }
361
362 if (permissions != null && !permissions.trim().isEmpty()) {
363 try {
364 String perms = stringPermissions(permissions, false);
365 if (!perms.isEmpty()) {
366 Files.setPosixFilePermissions(dotCopyFile, PosixFilePermissions.fromString(perms));
367 }
368 } catch (Exception e) {
369 logger.warn("Could not set file permissions to {} because {}", new Object[]{permissions, e});
370 }
371 }
372
373 if (owner != null && !owner.trim().isEmpty()) {
374 try {
375 UserPrincipalLookupService lookupService = dotCopyFile.getFileSystem().getUserPrincipalLookupService();
376 Files.setOwner(dotCopyFile, lookupService.lookupPrincipalByName(owner));
377 } catch (Exception e) {
378 logger.warn("Could not set file owner to {} because {}", new Object[]{owner, e});
379 }
380 }
381
382 if (group != null && !group.trim().isEmpty()) {
383 try {
384 UserPrincipalLookupService lookupService = dotCopyFile.getFileSystem().getUserPrincipalLookupService();
385 PosixFileAttributeView view = Files.getFileAttributeView(dotCopyFile, PosixFileAttributeView.class);
386 view.setGroup(lookupService.lookupPrincipalByGroupName(group));
387 } catch (Exception e) {
388 logger.warn("Could not set file group to {} because {}", new Object[]{group, e});
389 }
390 }
391
392 boolean renamed = false;
393 for (int i = 0; i < 10; i++) { // try rename up to 10 times.
394 if (dotCopyFile.toFile().renameTo(finalCopyFile.toFile())) { // rename file dotCopyFile is temple filename,
395 renamed = true;
396 break;// rename was successful
397 }
398 Thread.sleep(100L);// try waiting a few ms to let whatever might cause rename failure to resolve
399 }
400
401 if (!renamed) {
402 if (Files.exists(dotCopyFile) && dotCopyFile.toFile().delete()) {
403 logger.debug("Deleted dot copy file {}", new Object[]{dotCopyFile});
404 }
405 throw new ProcessException("Could not rename: " + dotCopyFile);
406 } else {
407 logger.info("Produced copy of {} at location {}", new Object[]{flowFile, finalCopyFile});
408 }
409
410 // for "reporting processor activity purpose"
411 session.getProvenanceReporter().send(flowFile, finalCopyFile.toFile().toURI().toString(), stopWatch.getElapsed(TimeUnit.MILLISECONDS));
412 session.transfer(flowFile, REL_SUCCESS);
413 } catch (final Throwable t) {
414 if (tempDotCopyFile != null) {
415 try {
416 Files.deleteIfExists(tempDotCopyFile);
417 } catch (final Exception e) {
418 logger.error("Unable to remove temporary file {} due to {}", new Object[]{tempDotCopyFile, e});
419 }
420 }
421
422 flowFile = session.penalize(flowFile);
423 logger.error("Penalizing {} and transferring to failure due to {}", new Object[]{flowFile, t});
424 session.transfer(flowFile, REL_FAILURE);
425 }
426 }
427
428 private long getFilesNumberInFolder(Path folder, String filename) {
429 String[] filesInFolder = folder.toFile().list();
430 return Arrays.stream(filesInFolder)
431 .filter(eachFilename -> !eachFilename.equals(filename))
432 .count();
433 }
434
435 // It's used to grant permission for the files
436 protected String stringPermissions(String perms, boolean directory) {
437 String permissions = "";
438 Matcher rwx = RWX_PATTERN.matcher(perms);
439 if (rwx.matches()) {
440 if (directory) {
441 // To read or write, directory access will be required
442 StringBuilder permBuilder = new StringBuilder();
443 permBuilder.append("$1");
444 permBuilder.append(rwx.group(1).equals("--") ? "$2" : "x");
445 permBuilder.append("$3");
446 permBuilder.append(rwx.group(3).equals("--") ? "$4" : "x");
447 permBuilder.append("$5");
448 permBuilder.append(rwx.group(5).equals("--") ? "$6" : "x");
449 permissions = rwx.replaceAll(permBuilder.toString());
450 } else {
451 permissions = perms;
452 }
453 } else if (NUM_PATTERN.matcher(perms).matches()) {
454 try {
455 int number = Integer.parseInt(perms, 8);
456 StringBuilder permBuilder = new StringBuilder();
457 if ((number & 0x100) > 0) {
458 permBuilder.append('r');
459 } else {
460 permBuilder.append('-');
461 }
462 if ((number & 0x80) > 0) {
463 permBuilder.append('w');
464 } else {
465 permBuilder.append('-');
466 }
467 if (directory || (number & 0x40) > 0) {
468 permBuilder.append('x');
469 } else {
470 permBuilder.append('-');
471 }
472 if ((number & 0x20) > 0) {
473 permBuilder.append('r');
474 } else {
475 permBuilder.append('-');
476 }
477 if ((number & 0x10) > 0) {
478 permBuilder.append('w');
479 } else {
480 permBuilder.append('-');
481 }
482 if ((number & 0x8) > 0) {
483 permBuilder.append('x');
484 } else {
485 if (directory && (number & 0x30) > 0) {
486 // To read or write, directory access will be required
487 permBuilder.append('x');
488 } else {
489 permBuilder.append('-');
490 }
491 }
492 if ((number & 0x4) > 0) {
493 permBuilder.append('r');
494 } else {
495 permBuilder.append('-');
496 }
497 if ((number & 0x2) > 0) {
498 permBuilder.append('w');
499 } else {
500 permBuilder.append('-');
501 }
502 if ((number & 0x1) > 0) {
503 permBuilder.append('x');
504 } else {
505 if (directory && (number & 0x6) > 0) {
506 // To read or write, directory access will be required
507 permBuilder.append('x');
508 } else {
509 permBuilder.append('-');
510 }
511 }
512 permissions = permBuilder.toString();
513 } catch (NumberFormatException ignore) {
514 }
515 }
516
517 return permissions;
518 }
519 }
