[Ocfs2-test-devel] [PATCH 3/4] Ocfs2-test: Add generic dir operation funcs for libocfs2test.
tristan.ye
tristan.ye at oracle.com
Tue Mar 31 19:33:42 PDT 2009
On Tue, 2009-03-31 at 16:51 -0700, Sunil Mushran wrote:
> The num_dirents is not efficient. As in, unlink does not
> make the space available for reuse. Now I understand we
> don't want to go overboard considering this is just for
> testing. But you could very easily use lists instead of arrays.
> See stringlist in debugs.ocfs2/utils.c as an example.
Actually the following dir operation funcs also copied from mark's
inline-dirs test:-)
Yes, I do agree with you on the point a list can save space when
unlinking compared to array during tests. but we also will be losing the
efficiency on random dir access, such as a random read, unlink and entry
find etc. Actaully, such random operations against dirents appears
everywhere during a dir related tests. I'm thinking if a list scheme
will bring us a obivious penality when the dirents number was
tremendous.
In fact a tests with a scale of 40000 entries take us 10M physical
memory, it's acceptable during the tests. but it does bring us the
efficiency on random access in return.
How do you think about it?
>
> Tristan Ye wrote:
> > Provide generic utility fuctions on dir operations for ocfs2-tests.
> >
> > Signed-off-by: Tristan Ye <tristan.ye at oracle.com>
> > ---
> > programs/libocfs2test/dir_ops.c | 345 +++++++++++++++++++++++++++++++++++++++
> > programs/libocfs2test/dir_ops.h | 85 ++++++++++
> > 2 files changed, 430 insertions(+), 0 deletions(-)
> > create mode 100644 programs/libocfs2test/dir_ops.c
> > create mode 100644 programs/libocfs2test/dir_ops.h
> >
> > diff --git a/programs/libocfs2test/dir_ops.c b/programs/libocfs2test/dir_ops.c
> > new file mode 100644
> > index 0000000..5857231
> > --- /dev/null
> > +++ b/programs/libocfs2test/dir_ops.c
> > @@ -0,0 +1,345 @@
> > +/* -*- mode: c; c-basic-offset: 8; -*-
> > + * vim: noexpandtab sw=8 ts=8 sts=0:
> > + *
> > + * dir_ops.c
> > + *
> > + * Provide generic utility fuctions on dir operations for ocfs2-tests
> > + *
> > + * Copyright (C) 2008 Oracle. All rights reserved.
> > + *
> > + * This program is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU General Public
> > + * License version 2 as published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> > + * General Public License for more details.
> > + */
> > +
> > +#include "dir_ops.h"
> > +
> > +extern unsigned long num_dirents;
> > +extern struct my_dirent *dirents;
> > +
> > +extern int is_dot_entry(struct my_dirent *dirent)
> > +{
> > + if (dirent->name_len == 1 && dirent->name[0] == '.')
> > + return 1;
> > + if (dirent->name_len == 2 && dirent->name[0] == '.'
> > + && dirent->name[1] == '.')
> > + return 1;
> > +
> > + return 0;
> > +}
> > +
> > +extern int unlink_dirent(char *dirname, struct my_dirent *dirent)
> > +{
> > + char path[PATH_MAX];
> > +
> > + dirent->name_len = 0;
> > + dirent->seen = 0;
> > + sprintf(path, "%s/%s", dirname, dirent->name);
> > +
> > + return unlink(path);
> > +}
> > +
> > +extern int create_and_prep_dir(char *dirname)
> > +{
> > + int ret;
> > + struct my_dirent *dirent;
> > +
> > + memset(dirents, 0, sizeof(struct my_dirent) * MAX_DIRENTS);
> > +
> > + dirent = &dirents[0];
> > + dirent->type = S_IFDIR >> S_SHIFT;
> > + dirent->name_len = 1;
> > + strcpy(dirent->name, ".");
> > +
> > + dirent = &dirents[1];
> > + dirent->type = S_IFDIR >> S_SHIFT;
> > + dirent->name_len = 2;
> > + strcpy(dirent->name, "..");
> > +
> > + num_dirents = 2;
> > + ret = mkdir(dirname, FILE_MODE);
> > + if (ret) {
> > + ret = errno;
> > + fprintf(stderr, "mkdir failure %d: %s\n", ret, strerror(ret));
> > + return ret;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int destroy_dir(char *dirname)
> > +{
> > + int ret, i;
> > + struct my_dirent *dirent;
> > +
> > + for (i = 0; i < num_dirents; i++) {
> > + dirent = &dirents[i];
> > +
> > + if (dirent->name_len == 0)
> > + continue;
> > +
> > + if (!is_dot_entry(dirent)) {
> > + ret = unlink_dirent(dirname, dirent);
> > + if (ret < 0) {
> > + ret = errno;
> > + fprintf(stderr, "unlink failure %d: %s\n", ret,
> > + strerror(ret));
> > + return ret;
> > + }
> > +
> > + dirent->name_len = 0;
> > + }
> > +
> > + }
> > +
> > + ret = rmdir(dirname);
> > + if (ret) {
> > + ret = errno;
> > + fprintf(stderr, "rmdir failure %d: %s\n", ret,
> > + strerror(ret));
> > + return ret;
> > + }
> > +
> > + num_dirents = 0;
> > +
> > + return 0;
> > +}
> > +
> > +extern struct my_dirent *find_my_dirent(char *name)
> > +{
> > + int i, len;
> > + struct my_dirent *my_dirent;
> > +
> > + len = strlen(name);
> > +
> > + for (i = 0; i < num_dirents; i++) {
> > + my_dirent = &dirents[i];
> > +
> > + if (my_dirent->name_len == 0)
> > + continue;
> > +
> > + if (my_dirent->name_len == len &&
> > + strcmp(my_dirent->name, name) == 0)
> > + return my_dirent;
> > + }
> > +
> > + return NULL;
> > +}
> > +
> > +
> > +extern int create_file(char *filename, char *dirname)
> > +{
> > + int ret, fd;
> > + struct my_dirent *dirent;
> > + char path[PATH_MAX];
> > +
> > + dirent = &dirents[num_dirents];
> > + num_dirents++;
> > +
> > + dirent->type = S_IFREG >> S_SHIFT;
> > + dirent->name_len = strlen(filename);
> > + dirent->seen = 0;
> > + strcpy(dirent->name, filename);
> > +
> > + sprintf(path, "%s/%s", dirname, dirent->name);
> > +
> > + fd = open(path, FILE_BUFFERED_RW_FLAGS, FILE_MODE);
> > + if (fd < 0) {
> > + ret = errno;
> > + fprintf(stderr, "Create file %s failure %d: %s\n", path, ret,
> > + strerror(ret));
> > + return ret;
> > + }
> > +
> > + close(fd);
> > +
> > + return 0;
> > +}
> > +
> > +extern int create_files(char *prefix, unsigned long num, char *dirname)
> > +{
> > + int i, ret;
> > + char dirent_nam[OCFS2_MAX_FILENAME_LEN];
> > + char path[PATH_MAX];
> > +
> > + for (i = 0; i < num; i++) {
> > + if (!prefix) {
> > + get_rand_nam(dirent_nam, 3, OCFS2_MAX_FILENAME_LEN/4);
> > + ret = create_file(dirent_nam, dirname);
> > + } else {
> > + snprintf(path, PATH_MAX, "%s%011d", prefix, i);
> > + ret = create_file(path, dirname);
> > + }
> > +
> > + if (ret < 0)
> > + return ret;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int is_dir_empty(char *name)
> > +{
> > + DIR *dir;
> > + int ret, entries = 0;
> > + struct dirent *dirent;
> > +
> > + dir = opendir(name);
> > + if (dir < 0) {
> > + ret = errno;
> > + fprintf(stderr, "dir open failure %d: %s\n", ret,
> > + strerror(ret));
> > + }
> > +
> > + dirent = readdir(dir);
> > + while (dirent) {
> > + entries++;
> > + dirent = readdir(dir);
> > + if ( entries > 2)
> > + break;
> > + }
> > +
> > + closedir(dir);
> > +
> > + if (entries == 2)
> > + return 1;
> > + else
> > + return 0;
> > +}
> > +
> > +/*
> > + *
> > + */
> > +extern int build_dir_tree(char *dirname, unsigned long entries,
> > + unsigned long depth, int is_random)
> > +{
> > + unsigned long i, dir_dirents, file_dirents;
> > + char fullpath[PATH_MAX];
> > + char dirent[OCFS2_MAX_FILENAME_LEN];
> > + unsigned long layer = depth;
> > +
> > + int fd, ret;
> > +
> > + ret = mkdir(dirname, FILE_MODE);
> > + if (ret < 0) {
> > + ret = errno;
> > + fprintf(stderr, "mkdir failure %d: %s\n", ret,
> > + strerror(ret));
> > + return ret;
> > + }
> > +
> > + if (layer == 0)
> > + return;
> > +
> > + if (is_random)
> > + dir_dirents = get_rand(1, entries - 1);
> > + else
> > + dir_dirents = entries / 2;
> > +
> > + file_dirents = entries - dir_dirents;
> > +
> > + for (i = 0; i < file_dirents; i++) {
> > + if (is_random) {
> > + get_rand_nam(dirent, 1, OCFS2_MAX_FILENAME_LEN - 20);
> > + snprintf(fullpath, PATH_MAX, "%s/%s%ld%ld",
> > + dirname, dirent, layer, i);
> > + } else
> > + snprintf(fullpath, PATH_MAX, "%s/%s%ld%ld",
> > + dirname, "F", layer, i);
> > + if (strlen(fullpath) > PATH_MAX)
> > + return;
> > + fd = open(fullpath, FILE_BUFFERED_RW_FLAGS, FILE_MODE);
> > + if (fd < 0) {
> > + ret = errno;
> > + fprintf(stderr, "create file failure %d: %s,"
> > + "filename = %s\n", ret, strerror(ret),
> > + fullpath);
> > + return ret;
> > + }
> > +
> > + close(fd);
> > + }
> > +
> > + for (i = 0; i < dir_dirents; i++) {
> > + if (is_random) {
> > + get_rand_nam(dirent, 1, OCFS2_MAX_FILENAME_LEN - 20);
> > + snprintf(fullpath, PATH_MAX, "%s/%s%ld%ld",
> > + dirname, dirent, layer, i);
> > + } else
> > + snprintf(fullpath, PATH_MAX, "%s/%s%ld%ld",
> > + dirname, "D", layer, i);
> > + if (strlen(fullpath) > PATH_MAX)
> > + return;
> > +
> > + build_dir_tree(fullpath, entries, layer - 1, is_random);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int set_semvalue(int sem_id)
> > +{
> > + union semun sem_union;
> > +
> > + sem_union.val = 1;
> > + if (semctl(sem_id, 0, SETVAL, sem_union) == -1) {
> > + perror("semctl");
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int semaphore_p(int sem_id)
> > +{
> > + struct sembuf sem_b;
> > +
> > + sem_b.sem_num = 0;
> > + sem_b.sem_op = -1; /* P() */
> > + sem_b.sem_flg = SEM_UNDO;
> > + if (semop(sem_id, &sem_b, 1) == -1) {
> > + fprintf(stderr, "semaphore_p failed\n");
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int semaphore_v(int sem_id)
> > +{
> > + struct sembuf sem_b;
> > +
> > + sem_b.sem_num = 0;
> > + sem_b.sem_op = 1; /* V() */
> > + sem_b.sem_flg = SEM_UNDO;
> > + if (semop(sem_id, &sem_b, 1) == -1) {
> > + fprintf(stderr, "semaphore_v failed\n");
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +extern int get_max_inlined_entries(int max_inline_size)
> > +{
> > + unsigned int almost_full_entries;
> > +
> > + /*
> > + Borrowed from mark's inline-dirs test to measure
> > + how to fill up the inlined directory.
> > + */
> > + almost_full_entries = max_inline_size / 512;
> > + almost_full_entries *= 512;
> > + almost_full_entries /= 32;
> > +
> > + almost_full_entries += 8;
> > +
> > +
> > + return almost_full_entries;
> > +}
> > +
> > diff --git a/programs/libocfs2test/dir_ops.h b/programs/libocfs2test/dir_ops.h
> > new file mode 100644
> > index 0000000..a20c181
> > --- /dev/null
> > +++ b/programs/libocfs2test/dir_ops.h
> > @@ -0,0 +1,85 @@
> > +/* -*- mode: c; c-basic-offset: 8; -*-
> > + * vim: noexpandtab sw=8 ts=8 sts=0:
> > + *
> > + * dir_ops.h
> > + *
> > + * Copyright (C) 2008 Oracle. All rights reserved.
> > + *
> > + * This program is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU General Public
> > + * License version 2 as published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> > + * General Public License for more details.
> > + */
> > +
> > +#ifndef DIR_OPS_H
> > +#define DIR_OPS_H
> > +
> > +#define _XOPEN_SOURCE 600
> > +#define _GNU_SOURCE
> > +#define _LARGEFILE64_SOURCE
> > +
> > +#include <sys/mman.h>
> > +#include <sys/types.h>
> > +#include <sys/stat.h>
> > +#include <sys/vfs.h>
> > +#include <sys/shm.h>
> > +#include <sys/sem.h>
> > +#include <unistd.h>
> > +#include <fcntl.h>
> > +#include <limits.h>
> > +
> > +#include <ocfs2/ocfs2.h>
> > +
> > +#include <dirent.h>
> > +
> > +#include <stddef.h>
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <errno.h>
> > +#include <string.h>
> > +
> > +#include <signal.h>
> > +#include <sys/wait.h>
> > +#include <inttypes.h>
> > +
> > +#define OCFS2_MAX_FILENAME_LEN 255
> > +#define MAX_DIRENTS 40000
> > +
> > +#define FILE_BUFFERED_RW_FLAGS (O_CREAT|O_RDWR|O_TRUNC)
> > +#define FILE_MODE (S_IRUSR|S_IWUSR|S_IXUSR|S_IROTH|\
> > + S_IWOTH|S_IXOTH|S_IRGRP|S_IWGRP|S_IXGRP)
> > +
> > +struct my_dirent {
> > + unsigned int type;
> > + unsigned int name_len;
> > + unsigned int seen;
> > + char name[OCFS2_MAX_FILENAME_LEN];
> > +};
> > +
> > +union semun {
> > + int val; /* value for SETVAL */
> > + struct semid_ds *buf; /* buffer for IPC_STAT, IPC_SET */
> > + unsigned short int *array; /* array for GETALL, SETALL */
> > + struct seminfo *__buf; /* buffer for IPC_INFO */
> > +};
> > +
> > +int is_dot_entry(struct my_dirent *dirent);
> > +int unlink_dirent(char *dirname, struct my_dirent *dirent);
> > +int create_and_prep_dir(char *dirname);
> > +int destroy_dir(char *dirname);
> > +struct my_dirent *find_my_dirent(char *name);
> > +int create_file(char *filename, char *dirname);
> > +int create_files(char *prefix, unsigned long num, char *dirname);
> > +int is_dir_empty(char *name);
> > +int build_dir_tree(char *dirname, unsigned long entries, unsigned long depth,
> > + int is_random);
> > +int set_semvalue(int sem_id);
> > +int semaphore_p(int sem_id);
> > +int semaphore_v(int sem_id);
> > +int get_max_inlined_entries(int max_inline_size);
> > +
> > +#endif
> >
>
More information about the Ocfs2-test-devel
mailing list