From 49f2490f6767268109ac45dbc2978573d5f54830 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 7 Oct 2008 15:46:57 +0530 Subject: Changes to Django db models (db re-init required) * The model structure is more logical now: - Release.provider (ForeignKey) - Release.archs (ManyToManyField) - Job.release (ForeignKey) - No Job.{arch,provider}; implicit in Job.release - Stage() no longer around; Job.stage = models.CharField instead - Arch() -> unique_together = ['specific', 'generic'] * The db's initialisation and db_defaults.py also reflect this * The Form filtering will now be done on the JS side. How is yet undecided. --- master/custom/db_defaults.py | 74 +++++++++++++++++++++++--------------- master/custom/sample_data.py | 5 +-- master/master/const.py | 20 +++++++++++ master/master/models.py | 85 +++++++++++++++++++++----------------------- master/setup-master.py | 55 ++++++++++++---------------- 5 files changed, 131 insertions(+), 108 deletions(-) (limited to 'master') diff --git a/master/custom/db_defaults.py b/master/custom/db_defaults.py index 2af2171..c8113b7 100644 --- a/master/custom/db_defaults.py +++ b/master/custom/db_defaults.py @@ -8,38 +8,56 @@ # Database initialization values -## User-related defaults -groups = ('dev', 'at', 'user',) +########################### +## User-related defaults ## +########################### +groups = ('admin', 'dev', 'at', 'user',) -## Job-related defaults +########################## +## Job-related defaults ## +########################## providers = ('gentoo', 'funtoo',) -# No 'arm' in here because the dir structure is weird -# and it hasn't been updated in forever anyway. -# Use a custom stage url if you want to use arm -# -# 'Mirror dir': () -archs = { 'gentoo': (('x86', ('i686', 'x86',)), - ('amd64', ('amd64',)), - ('ppc', ('970-32ul', '970-64ul', 'g4', - 'power5', 'ppc', 'ppc64',)), - ('sparc', ('sparc64',)), - ('alpha', ('alpha',)), - ('ia64', ('ia64',)), - ('hppa', ('hppa1.1', 'hppa2.0',)), - ('mips', ('cobalt', 'mips3', 'mips4', - 'n32',)), - ('sh', ('sh4',)), - ('s390', ('s390', 's390x',)),), - 'funtoo': ('amd64', 'athlon-xp', 'core32', 'core64', - 'i686', 'openvz', 'pentium4', 'x86', '~amd64', - '~x86'), } -stages = { 'gentoo': ('stage1', 'stage2', 'stage3',), - 'funtoo': ('stage1', 'stage2', 'stage3',), } +# All archs are listed here +# 'generic arch': ('specific_arch1', 'specific_arch2',) +archs = {'x86': ('i686', 'x86',), + 'amd64': ('amd64',), + 'ppc': ('970-32ul', '970-64ul', 'g4', 'power5', 'ppc', + 'ppc64',), + 'sparc': ('sparc64',), + 'alpha': ('alpha',), + 'ia64': ('ia64',), + 'hppa': ('hppa1.1', 'hppa2.0',), + 'mips': ('cobalt', 'mips3', 'mips4', 'n32',), + 'sh': ('sh4',), + 's390': ('s390', 's390x',), + '': ('athlon-xp', 'core32', 'core64', 'i686', + 'openvz', 'pentium4', '~amd64', '~x86',),} -# Ask drobbins to have a 'latest' symlink. -releases = {'gentoo': ('2007.0', '2008.0'), - 'funtoo': ('latest',)} +# Gentoo archs which don't have a >=2007.0 release aren't listed +# Use a custom stage url if you want to use them +# +# Funtoo has only specific archs => generic archs are blank +releases = {'gentoo': {'2007.0': {'x86': archs['x86'], + 'amd64': archs['amd64'], + 'ppc': ('ppc', 'ppc64',), + 'sparc': archs['sparc'], + 'alpha': archs['alpha'], + 'ia64': archs['ia64'], + 'hppa': archs['hppa'], + 'mips': ('mips3', 'mips4',), + 'sh': archs['sh'], + 's390': archs['s390'],}, + '2008.0': {'x86': archs['x86'], + 'amd64': archs['amd64'], + 'ppc': archs['ppc'], + 'sparc': archs['sparc'], + 'alpha': archs['alpha'], + 'ia64': archs['ia64'], + 'hppa': archs['hppa'], + 'mips': archs['mips'],},}, + # Ask drobbins to have a 'latest' symlink. + 'funtoo': {'latest': {'': archs['']}},} mirrors = { 'gentoo': {'servers': (('http://gentoo.osuosl.org/', 'releases/'), ('http://ftp.jaist.ac.jp/', 'pub/Linux/Gentoo/releases/'),), diff --git a/master/custom/sample_data.py b/master/custom/sample_data.py index c7fb3f4..0e91871 100644 --- a/master/custom/sample_data.py +++ b/master/custom/sample_data.py @@ -8,9 +8,10 @@ sample_job = { 'name': 'Sample AutotuA job', - 'maintainer': ('test_user', 'test_user_email@test.com', None), + 'maintainer': ('test_user', 'test_user_email@test.com', 'password'), 'provider': 'gentoo', - 'stage': 'stage3', + # If stage is unspecified, defaults to stage3 + #'stage': 'stage3', 'arch': 'i686', 'release': '2008.0', 'jobtagerev': '', diff --git a/master/master/const.py b/master/master/const.py index 9c6ef4a..abdd4e4 100644 --- a/master/master/const.py +++ b/master/master/const.py @@ -11,3 +11,23 @@ from autotua import config # FIXME: insecure tmpdir. TMPDIR = '/tmp/master' JOBTAGE = config.JOBTAGE_DIR + +STAGE_TYPES = ( ('stage1', 'stage1'), + ('stage2', 'stage2'), + ('stage3', 'stage3'), + ('stage4', 'stage4'),) + +# States +JOB_STATES = ( ('R', 'Ready'), + ('Q', 'Queued'), + ('W', 'Working'), + ('P', 'Paused'), + ('C', 'Cancelled'), + ('E', 'Errored out'), + ('D', 'Done'), + ('U', 'Unknown'),) + +SLAVE_STATES = ( ('I', 'Idle'), + ('W', 'Working'), + ('N', 'Not reachable'), + ('U', 'Unknown'),) diff --git a/master/master/models.py b/master/master/models.py index 06926f9..6cf9330 100644 --- a/master/master/models.py +++ b/master/master/models.py @@ -15,25 +15,27 @@ import const, random, urllib2 #################### ### Models begin ### #################### -class StageProvider(models.Model): - # Identifier for the stage provider. - # Ex: 'gentoo', 'funtoo', 'gentooexperimental' - name = models.CharField(max_length=25, unique=True) - def __unicode__(self): - return self.name +class Slave(models.Model): + name = models.CharField(max_length=30, unique=True) + # User which owns this slave + owner = models.ForeignKey(User) + # Groups which can use this slave + users = models.ManyToManyField(Group) + # Status of slave + state = models.CharField(max_length=30, default='U', + choices=const.SLAVE_STATES) -class Stage(models.Model): - # stage{1,2,3,4} etc - name = models.CharField(max_length=25) - # Stage provider that supports this stage - provider = models.ForeignKey(StageProvider) + def __unicode__(self): + return "%s (%s)" % (self.name, self.state) - class Meta: - unique_together = ['name', 'provider'] +class Provider(models.Model): + # Identifier for the mirror provider. + # Ex: 'gentoo', 'funtoo', 'gentooexperimental' + name = models.CharField(max_length=25, unique=True) def __unicode__(self): - return '%s (%s)' % (self.name, self.provider.name) + return self.name class Arch(models.Model): # Specific arch i686, g4, ~amd64, core32 etc. @@ -42,20 +44,20 @@ class Arch(models.Model): # Ex: 'specific': 'i686', 'generic': 'x86' # This is needed for Gentoo mirrors generic = models.CharField(max_length=25, blank=True) - # Stage provider that supports stages with this arch - provider = models.ForeignKey(StageProvider) class Meta: - unique_together = ['specific', 'provider'] + unique_together = ['specific', 'generic'] def __unicode__(self): - return '%s:%s (%s)' % (self.generic, self.specific, self.provider.name) + return '%s (%s)' % (self.specific, self.generic) class Release(models.Model): # Release number/identifier name = models.CharField(max_length=25) # Release provider - provider = models.ForeignKey(StageProvider) + provider = models.ForeignKey(Provider) + # Archs supported for this release + archs = models.ManyToManyField(Arch) class Meta: unique_together = ['name', 'provider'] @@ -65,7 +67,7 @@ class Release(models.Model): class Mirror(models.Model): # Mirror owner - owner = models.ForeignKey(StageProvider) + owner = models.ForeignKey(Provider) # Mirror root URL # Ex: 'http://gentoo.osousl.org/', 'http://www.funtoo.org/' server = models.URLField() @@ -73,7 +75,7 @@ class Mirror(models.Model): # Ex: 'releases/', 'linux/' prefix = models.CharField(max_length=25) # How to get a URI for a stage file by putting in various variables - # Ex: Gentoo mirrors are structured as follows: + # Ex: Gentoo mirrors are structured as follows: # '%(gen_arch)s/%(release)s/stages/%(stage)s-%(arch)s-%(release)s.tar.bz2' # Funtoo is structured as follows: # '%(arch)s/%(owner)s-%(arch)s-%(release)/%(stage)s-%(arch)s-%(release)s.tar.bz2' @@ -98,20 +100,25 @@ class Job(models.Model): help_text="Identifier for the job") # Name of maintainer maintainer = models.ForeignKey(User, editable=False) - # Stage provider - provider = models.ForeignKey(StageProvider) + # Slaves running the job + slaves = models.ManyToManyField(Slave) # Stage type - stage = models.ForeignKey(Stage) - # i686, amd64, g4, etc. - arch = models.ForeignKey(Arch) - # Release + stage = models.CharField(max_length=30, default="stage3", + choices=const.STAGE_TYPES) + # Release (gives implicit info about provider) release = models.ForeignKey(Release) + # i686, amd64, g4, etc. + arch = models.ForeignKey(Arch, limit_choices_to=release.archs.all()) # Type of stage; hardened, uclibc, etc #type = CharField(maxlength=25) # const.stage_types # Revision of jobtage tree to use (not to be entered by user) jobtagerev = models.CharField(max_length=50, editable=False) # Root jobuild atom; auto-converted to deplist atoms = models.TextField() + # State of the job + state = models.CharField(max_length=30, unique=True, + choices=const.JOB_STATES, + default='U', editable=False) class Meta: unique_together = ['name', 'maintainer'] @@ -126,7 +133,7 @@ class Job(models.Model): def _get_deplist(self, atoms, rev): """ - Use autotua-slave to get the deplist + Use autotua-slave to get the deplist from the root jobuild atom Support multiple atoms for the multiple-save case @@ -148,28 +155,16 @@ class Job(models.Model): # Fake jobuild.Resolver to get at _unique ;p return jobuild.Resolver(None)._unique(deplist) -class Slave(models.Model): - # Which user owns this slave - owner = models.ForeignKey(User) - # Status of slave - #status = models.ForeignKey(Job) - ################### ### Forms begin ### ################### class JobForm(ModelForm): - # FIXME: Make the default here as 'gentoo' once db is populated. - provider = ModelChoiceField(label="Stage Provider", - queryset=StageProvider.objects.all()) - # FIXME FIXME FIXME: All fields after this should filter on the - #basis of the above specified provider. - # Javascript? Two steps for form submission? - stage = ModelChoiceField(label="Stage type", - queryset=Stage.objects.all()) - arch = ModelChoiceField(label="Arch", - queryset=Arch.objects.all()) + # FIXME: Arch should filter on the basis of the Release + # Javascript logic would be best/easiest. release = ModelChoiceField(label="Release", - queryset=Release.objects.all()) + queryset=Release.objects.all()) + arch = ModelChoiceField(label="Arch", + queryset=Arch.objects.all()) class Meta: model = Job include = ('name', 'atoms') diff --git a/master/setup-master.py b/master/setup-master.py index 4f66699..5bfcfef 100755 --- a/master/setup-master.py +++ b/master/setup-master.py @@ -72,7 +72,7 @@ def install_master(): def syncdb_master(): """Initialize the database""" - from db_defaults import groups, providers, archs, stages, releases, mirrors + from db_defaults import groups, providers, archs, releases, mirrors import copy management.call_command('syncdb') @@ -80,34 +80,26 @@ def syncdb_master(): for group in groups: Group(name=group).save() # Job-related defaults + for (gen_arch, spec_archs) in archs.iteritems(): + gen_archobj = Arch(generic=gen_arch) + for spec_arch in spec_archs: + archobj = copy.copy(gen_archobj) + archobj.specific = spec_arch + archobj.save() + # Populate providers for provider in providers: - provobj = StageProvider(name=provider) + provobj = Provider(name=provider) provobj.save() - # Populate arch list - for generic_arch in archs[provider]: - gen_archobj = Arch(provider=provobj) - # If (generic, (specific1, specific2)) - if isinstance(generic_arch, tuple): - gen_archobj.generic = generic_arch[0] - for arch in generic_arch[1]: - archobj = copy.copy(gen_archobj) - archobj.specific = arch - archobj.save() - # If (specific1, specific2, specific3) - else: - gen_archobj.generic = generic_arch - gen_archobj.specific = generic_arch - gen_archobj.save() - # Populate stage list - for stage in stages[provider]: - stageobj = Stage(provider=provobj) - stageobj.name = stage - stageobj.save() # Populate release list - for release in releases[provider]: - releaseobj = Release(provider=provobj) - releaseobj.name = release - releaseobj.save() + releaseobj = Release(provider=provobj) + for (release, relarchs) in releases[provider].iteritems(): + relobj = copy.copy(releaseobj) + relobj.name = release + relobj.save() + for (relgen_arch, relspec_archs) in relarchs.iteritems(): + for relspec_arch in relspec_archs: + relobj.archs.add(Arch.objects.get(generic=relgen_arch, + specific=relspec_arch)) # Populate mirror list obj = Mirror(owner=provobj) obj.structure = mirrors[provider]['structure'] @@ -127,13 +119,10 @@ def setup_sample_job(): job.maintainer = maintainers[0] else: job.maintainer = User.objects.create_user(*sample_job['maintainer']) - job.provider = StageProvider.objects.get(name=sample_job['provider']) - job.stage = Stage.objects.get(name=sample_job['stage'], - provider=job.provider) - job.arch = Arch.objects.get(specific=sample_job['arch'], - provider=job.provider) job.release = Release.objects.get(name=sample_job['release'], - provider=job.provider) + provider=Provider.objects.get(name=sample_job['provider'])) + job.arch = job.release.archs.get(specific=sample_job['arch']) + # If stage is not set, defaults to stage3 job.save() if len(sys.argv) < 3: @@ -163,7 +152,7 @@ elif sys.argv[1] == 'syncdb': import settings from django.core.management import setup_environ setup_environ(settings) - from master.models import User, Group, StageProvider, Arch, Stage, Release, Mirror, Job + from master.models import User, Group, Provider, Arch, Release, Mirror, Job, Slave # Start stuff syncdb_master() setup_sample_job() -- cgit v1.2.3-65-gdbad